diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev index 2c33440d4e2..59a585ec262 100644 --- a/docker/.ngc_version.dev +++ b/docker/.ngc_version.dev @@ -1 +1 @@ -nvcr.io/nvidia/pytorch:26.02-py3 \ No newline at end of file +nvcr.io/nvidia/pytorch:26.04-py3 \ No newline at end of file diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md new file mode 100644 index 00000000000..189aa540f82 --- /dev/null +++ b/examples/moe_recipes/README.md @@ -0,0 +1,127 @@ +# MoE Recipes + +This directory contains self-contained MoE training recipes. Each YAML file includes: + +- `DEPENDENCIES`: PyTorch base image and Dockerfile content used for the recipe. +- `ENV_VARS`: Environment variables expected by the runtime. +- `ARGS`: Megatron-LM training arguments. + +## Recipe Index + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelRecipeGPUsTP/PP/EP/CP/ETPMBS/GBS/SLFeatures
DeepSeek-V3B200 MXFP82561/8/32/1/11/2048/4096DeepEP; EP overlap
GB200 MXFP82561/4/64/1/11/8192/4096HybridEP; partial CG; EP overlap; offload
GB300 MXFP82561/4/64/1/11/8192/4096HybridEP; partial CG; EP overlap
H100 BF1610241/16/64/1/11/8192/4096BF16 baseline
H100 FP810242/8/64/1/11/8192/4096DeepEP; EP overlap
Qwen3-235B-A22BGB200 MXFP8 full CG1281/1/64/1/11/8192/4096paged stash; full CG; HybridEP; EP overlap
GB200 MXFP8 partial CG1281/1/64/1/11/8192/4096partial CG; HybridEP; EP overlap
GB300 MXFP8 full CG1281/1/64/1/11/8192/4096paged stash; full CG; HybridEP; EP overlap
H100 BF162562/8/32/1/11/2048/4096router/preprocess CG; HybridEP; EP overlap
Qwen3-30B-A3BH100 FP8321/1/8/1/11/256/4096FP8 blockwise; router/preprocess CG
H100 BF16321/1/8/1/11/256/4096BF16 baseline
GB200 BF16161/1/16/1/14/512/4096BF16 baseline
GB200 MXFP8 partial CG161/1/16/1/14/512/4096MXFP8; partial CG
GB200 MXFP8 paged stash161/1/16/1/14/512/4096MXFP8; paged stash; full CG
+ +Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size, SL = sequence length, CG = CUDA graph. The tuple orders are `TP/PP/EP/CP/ETP` and `MBS/GBS/SL`. diff --git a/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml new file mode 100644 index 00000000000..0979f7ad6ab --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml @@ -0,0 +1,216 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: b300-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, cuDNN frontend, and FLA for + # Qwen3.5 Gated Delta Net support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" && \ + pip install --no-cache-dir fla-core==0.4.2 flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 32 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: deepep + moe_deepep_num_sms: 24 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + - mlp + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 2048 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-B200-MXFP8-TP1PP8EP32-GBS2048 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml new file mode 100644 index 00000000000..9f81ca049b7 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml @@ -0,0 +1,271 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE commit 2026-04-22 + ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts + # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently + # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image + # with --target base to skip the hybridep stage below. + ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f" + RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \ + cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \ + rm -rf /root/.cache /tmp/* + + # ========================= + # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in + # nixl.h, which isn't available in this container. Kept for reference; do NOT + # build with `--target hybridep` until NIXL is staged into the build context. + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP commit pinned (2026-04-16) + cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + NCCL_GRAPH_REGISTER: 0 + NVTE_CPU_OFFLOAD_V1: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + NUM_OF_STAGES_DISPATCH_API: '10' + NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 4 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_hybridep_num_sms: 32 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + offload_optimizer_states: true + fine_grained_activation_offloading: true + offload_modules: + - expert_fc1 + delay_offload_until_cuda_graph: true + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + cuda_graph_warmup_steps: 1 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-GB200-MXFP8-TP1PP4EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml new file mode 100644 index 00000000000..6727da78bc7 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml @@ -0,0 +1,262 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE commit 2026-04-22 + ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts + # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently + # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image + # with --target base to skip the hybridep stage below. + ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f" + RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \ + cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \ + rm -rf /root/.cache /tmp/* + + # ========================= + # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in + # nixl.h, which isn't available in this container. Kept for reference; do NOT + # build with `--target hybridep` until NIXL is staged into the build context. + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP commit pinned (2026-04-16) + cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: 0 + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + NUM_OF_STAGES_DISPATCH_API: '10' + NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 4 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_hybridep_num_sms: 32 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + cuda_graph_warmup_steps: 1 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-GB300-MXFP8-TP1PP4EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml new file mode 100644 index 00000000000..bc35ea5f510 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml @@ -0,0 +1,192 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 16 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_enable_deepep: true + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-TP1PP16EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml new file mode 100644 index 00000000000..9fcc2696c96 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml @@ -0,0 +1,207 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*3|(tt|)*29m|L + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: deepep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + - mlp + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: blockwise + fp8_format: e4m3 + fp8_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_fp8: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-FP8-TP2PP8EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml new file mode 100644 index 00000000000..a9581dc95e8 --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml @@ -0,0 +1,259 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604-te215 + # + # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and + # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to + # Dockerfile.GB200.GB300.torch2604. + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE v2.15 main + ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support + # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP / + # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \ + pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker) + cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 + rm -f .use_nixl + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + NVLINK_DOMAIN_SIZE: '72' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + USE_TE_OPS: 'True' + CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128' + NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + PYTHONWARNINGS: ignore + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.2 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.0 + moe_paged_stash_buffer_size_factor_cpu: 0.8 + use_transformer_engine_op_fuser: true + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + cuda_graph_warmup_steps: 2 + no_check_for_nan_in_loss_and_grad: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 268554688 + exit_duration_in_mins: 230 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB200-MXFP8-PagedStash-FullCG-Overlap-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml new file mode 100644 index 00000000000..89f1bab705d --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml @@ -0,0 +1,219 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + NVLINK_DOMAIN_SIZE: '72' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + USE_TE_OPS: 'True' + CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128' + NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + PYTHONWARNINGS: ignore +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + cuda_graph_warmup_steps: 2 + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 268554688 + exit_duration_in_mins: 230 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB200-MXFP8-PartialCG-Overlap-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml new file mode 100644 index 00000000000..2b2d42e4494 --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml @@ -0,0 +1,247 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604-te215 + # + # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and + # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to + # Dockerfile.GB200.GB300.torch2604. + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE v2.15 main + ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support + # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP / + # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \ + pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker) + cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 + rm -f .use_nixl + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + USE_MNNVL: '1' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + PYTHONWARNINGS: ignore + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.2 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.0 + use_transformer_engine_op_fuser: true + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + cuda_graph_warmup_steps: 2 + no_check_for_nan_in_loss_and_grad: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_iters: 20 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB300-MXFP8-PagedStash-FullCG-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml new file mode 100644 index 00000000000..d2ebd988f8d --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml @@ -0,0 +1,186 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 32 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + num_layers_per_virtual_pipeline_stage: 3 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + recompute_granularity: selective + recompute_modules: moe_act layernorm + cuda_graph_impl: transformer_engine + cuda_graph_scope: moe_router moe_preprocess + distributed_timeout_minutes: 220 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 2048 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-235B-TP2PP8EP32-GBS2048 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml b/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml new file mode 100644 index 00000000000..b4a678b9223 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml @@ -0,0 +1,181 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-BF16-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml new file mode 100644 index 00000000000..694373bbaad --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml @@ -0,0 +1,195 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: 0 + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_transformer_engine_op_fuser: true + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.5 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.1 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 20 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-MXFP8-PagedStash-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml new file mode 100644 index 00000000000..9a03735e782 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml @@ -0,0 +1,190 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + external_cuda_graph: true + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-MXFP8-PartialCG-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml b/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml new file mode 100644 index 00000000000..a55ac6434c5 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml @@ -0,0 +1,176 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 8 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 256 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-TP1PP1EP8-GBS256 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml b/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml new file mode 100644 index 00000000000..2954da5387f --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml @@ -0,0 +1,190 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 8 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + use_mcore_models: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 256 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: blockwise + fp8_format: e4m3 + fp8_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-FP8-PartialCG-TP1PP1EP8-GBS256 + enable_experimental: true diff --git a/examples/multimodal_dev/README.md b/examples/multimodal_dev/README.md new file mode 100644 index 00000000000..e4e6c53ceb4 --- /dev/null +++ b/examples/multimodal_dev/README.md @@ -0,0 +1,222 @@ +# multimodal_dev — Standalone Multimodal Training + +Standalone, model-agnostic training entry point for multimodal +vision-language models built on Megatron-Core (FSDP + EP). + +## Directory Structure + +``` +multimodal_dev/ +├── pretrain_multimodal.py # Training entry point (model-agnostic) +├── forward_step.py # Forward step, TP broadcast, loss computation +├── arguments.py # Multimodal CLI arguments +├── data/ +│ └── mock.py # Mock dataset for end-to-end testing +├── models/ +│ ├── __init__.py # MODEL_REGISTRY — central model registry +│ ├── base.py # MultimodalModel base class (vision encoder + GPTModel) +│ └── qwen35_vl/ # Qwen3.5-VL architecture +│ ├── factory.py # Factory functions for pretrain entry point +│ ├── model.py # Qwen35VLModel (MRoPE, vision encoder wiring) +│ ├── configuration.py # TransformerConfig builders and constants +│ ├── specs.py # Layer spec builders (hybrid attention, ViT) +│ ├── mrope.py # 3D MRoPE position ID computation +│ └── vision_encoder.py# ViT encoder (patch embed, merger, RoPE) +└── scripts/ # Launch scripts (torchrun, Slurm) +``` + +## Quick Start + +```bash +torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \ + --model-arch qwen35_vl \ + --dataset-provider mock \ + ... # other Megatron args (--num-layers, --hidden-size, etc.) +``` + +## Checkpoint Conversion (HF → Megatron-FSDP DTensor) + +Convert a HuggingFace release to a Megatron-FSDP DTensor checkpoint via +[Megatron-Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge) before +pretraining from pretrained weights. + +### Setup + +Clone Bridge and pin its `3rdparty/Megatron-LM` submodule to this branch: + +```bash +git clone --recurse-submodules https://github.com/NVIDIA-NeMo/Megatron-Bridge.git +cd Megatron-Bridge/3rdparty/Megatron-LM +git remote add wplf https://github.com/wplf/Megatron-LM.git +git fetch wplf feat/qwen35-vl-example +git checkout feat/qwen35-vl-example +cd ../.. +``` + +### Convert + +Single 8×GPU node, EP=8 / TP=CP=1; substitute any Qwen3.5 variant for +`--hf-model`: + +```bash +PYTHONPATH=./src:./3rdparty/Megatron-LM/ \ + torchrun --nproc_per_node=8 \ + examples/conversion/mfsdp/convert_checkpoints_fsdp.py import \ + --hf-model Qwen/Qwen3.5-35B-A3B \ + --megatron-path ${WORKSPACE}/models/Qwen/Qwen3.5-35B-A3B-fsdp \ + --ckpt-format fsdp_dtensor \ + --ep 8 +``` + +HF weights are auto-fetched on first run via `huggingface_hub`. Adjust +`--tp` / `--cp` / `--ep` to match the training topology (must satisfy +`WORLD_SIZE % (TP*CP*EP) == 0`). + +### Output + +``` +${WORKSPACE}/models/Qwen/Qwen3.5-35B-A3B-fsdp/ +├── iter_0000000/ +│ ├── __0_0.distcp .. __7_0.distcp # FSDP DTensor shards, one per rank (~18 GB each for 35B-A3B) +│ ├── .metadata +│ ├── run_config.yaml +│ └── train_state.pt +├── latest_checkpointed_iteration.txt +└── latest_train_state.pt +``` + +### Bridge dependency + +Requires +[NVIDIA-NeMo/Megatron-Bridge#3987](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/3987) +(skip tokenizer save). Without that fix the checkpoint is still written +correctly but the script exits non-zero after save with +`AttributeError: 'TokenizerConfig' object has no attribute 'make_vocab_size_divisible_by'` +against this branch's `megatron.core.tokenizers.utils.build_tokenizer`. + +## Architecture + +`pretrain_multimodal.py` is **model-agnostic**. All model-specific logic +is delegated to factory functions registered in `MODEL_REGISTRY` +(`models/__init__.py`). The entry point handles only generic concerns: + +- Building `language_config` from Megatron CLI args +- Constructing `vision_config` via the registry +- Applying vision recompute and dtype propagation +- Routing to model and dataset factories + +The `forward_step` is also model-agnostic — it uses the model's +`compute_position_ids()` method polymorphically and passes a standard +batch dict. + +## Adding a New Model Architecture + +Adding a new model (e.g. `llava_next`) requires **no changes** to +`pretrain_multimodal.py` or `forward_step.py`. Follow these steps: + +### Step 1 — Create the model package + +``` +multimodal_dev/models/llava_next/ +├── __init__.py +├── factory.py # Required: factory functions +├── configuration.py # Vision/language TransformerConfig builders +├── model.py # Model class (subclass MultimodalModel) +├── specs.py # Layer spec builders +└── vision_encoder.py # Vision encoder (if custom) +``` + +### Step 2 — Implement factory functions + +Create `factory.py` with up to three functions: + +```python +# models/llava_next/factory.py + +def post_language_config(language_config, args): + """(Optional) Mutate language_config with model-specific fields.""" + # e.g. language_config.some_field = value + pass + +def set_vision_flops_metadata(args, language_config, vision_config): + """(Optional) Set vision FLOPs metadata on args.""" + args.count_vision_model_flops = True + args.vision_flops_variant = "llava_next" + # ... set dimension fields for FLOPs calculation + +def build_model(args, language_config, vision_config, **kwargs): + """(Required) Build and return the complete model instance.""" + from .model import LlavaNextModel + from .specs import get_llava_next_language_spec + + language_spec = get_llava_next_language_spec( + config=language_config, + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + return LlavaNextModel( + language_config=language_config, + language_spec=language_spec, + vision_config=vision_config, + # ... model-specific args + ) +``` + +### Step 3 — Register in `MODEL_REGISTRY` + +Add an entry in `models/__init__.py`: + +```python +from multimodal_dev.models.llava_next.configuration import ( + get_llava_next_vision_config, +) +from multimodal_dev.models.llava_next.factory import ( + build_model as _build_llava_next_model, + post_language_config as _llava_next_post_language_config, + set_vision_flops_metadata as _llava_next_vision_flops, +) + +MODEL_REGISTRY["llava_next"] = { + "model_factory_fn": _build_llava_next_model, # required + "vision_config_fn": get_llava_next_vision_config, # required + "post_language_config_fn": _llava_next_post_language_config, # optional + "vision_flops_fn": _llava_next_vision_flops, # optional + "dataset_providers": { # optional + "mock": "multimodal_dev.data.llava_mock.train_valid_test_datasets_provider", + }, +} +``` + +### Step 4 — (Optional) Add a dataset provider + +Create a dataset module under `data/` if the model needs custom data +preprocessing. The provider function signature is: + +```python +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Return (train_dataset, val_dataset, test_dataset).""" + ... +``` + +Register it in the `dataset_providers` dict of the registry entry. +Providers can be either direct callables or dotted import path strings +(resolved lazily at runtime). + +### Step 5 — Launch + +```bash +torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \ + --model-arch llava_next \ + --dataset-provider mock \ + ... +``` + +## Registry Entry Reference + +| Field | Required | Signature | +|-------|----------|-----------| +| `model_factory_fn` | Yes | `(args, language_config, vision_config, **kwargs) -> MegatronModule` | +| `vision_config_fn` | Yes | `(num_layers_override=None) -> TransformerConfig` | +| `post_language_config_fn` | No | `(language_config, args) -> None` | +| `vision_flops_fn` | No | `(args, language_config, vision_config) -> None` | +| `dataset_providers` | No | `Dict[str, str \| callable]` | diff --git a/examples/multimodal_dev/__init__.py b/examples/multimodal_dev/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/arguments.py b/examples/multimodal_dev/arguments.py new file mode 100644 index 00000000000..35655831bfb --- /dev/null +++ b/examples/multimodal_dev/arguments.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Extra CLI arguments for multimodal_dev standalone training.""" + + +def add_multimodal_args(parser): + """Add multimodal-specific arguments to the Megatron argument parser.""" + group = parser.add_argument_group( + "Multimodal", "Multimodal model arguments", + ) + + group.add_argument( + "--model-arch", + type=str, + default="qwen35_vl", + help="Model architecture. Available: qwen35_vl", + ) + group.add_argument( + "--model-variant", + type=str, + default="proxy", + help="Model variant (size). E.g. proxy, 9b, 397b_a17b", + ) + group.add_argument( + "--dataset-provider", + type=str, + default="mock", + help="Dataset provider: mock", + ) + group.add_argument( + "--image-token-id", + type=int, + default=248056, + help="Token ID for image placeholder tokens", + ) + group.add_argument( + "--image-size", + type=int, + default=224, + help="Image size (height and width) for mock data", + ) + group.add_argument( + "--total-seq-length", + type=int, + default=1024, + help="Total sequence length for mock data", + ) + group.add_argument( + "--image-seq-length", + type=int, + default=256, + help="Number of image tokens in mock data", + ) + group.add_argument( + "--vision-num-layers", + type=int, + default=None, + help=( + "Override for vision backbone depth. " + "Useful for proxy perf runs." + ), + ) + group.add_argument( + "--hf-processor-path", + type=str, + default=None, + help=( + "HuggingFace processor path for real VLM datasets " + "(e.g. Qwen/Qwen2.5-VL-7B-Instruct)" + ), + ) + group.add_argument( + "--recompute-vision", + action="store_true", + default=False, + help=( + "Enable full activation recomputation for vision encoder layers. " + "Uses uniform method and recomputes every layer. " + "Independent of the decoder --recompute-* flags." + ), + ) + group.add_argument( + "--use-packed-sequence", + action="store_true", + default=False, + help=( + "Pack variable-length sequences into THD format to eliminate " + "padding waste." + ), + ) + group.add_argument( + "--use-vanilla-collate-fn", + action="store_true", + default=False, + help=( + "Use vanilla collate function to collate the data." + ), + ) + + return parser diff --git a/examples/multimodal_dev/data/__init__.py b/examples/multimodal_dev/data/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/data/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/data/cord_v2.py b/examples/multimodal_dev/data/cord_v2.py new file mode 100644 index 00000000000..69fd4c13ec4 --- /dev/null +++ b/examples/multimodal_dev/data/cord_v2.py @@ -0,0 +1,397 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""CORD-V2 VLM dataset for multimodal_dev training. + +Single-turn image-text dataset using a HuggingFace ``AutoProcessor`` for +tokenization and image preprocessing. This module is the reference +implementation for the CORD-V2 receipt-OCR dataset. No multi-turn support — +each sample is one image + question → answer pair. + +Each image is preprocessed via ``qwen_vl_utils.process_vision_info`` and +fed to the processor with Qwen-VL's recommended ``min_pixels`` / +``max_pixels`` budget, so the per-sample patch grid varies with aspect +ratio. The run script must therefore pass ``--use-vanilla-collate-fn`` +(which the example launcher does) so the dataloader does not try to stack +variable-shape tensors. + +Usage:: + + torchrun ... pretrain_multimodal.py \\ + --model-arch qwen35_vl --dataset-provider cord_v2 \\ + --hf-processor-path Qwen/Qwen3.5-397B-A17B \\ + --total-seq-length 4096 --use-vanilla-collate-fn + +Adding another VLM dataset +-------------------------- + +The dataset layer mirrors the model layer's registry pattern: each dataset +ships its own module and a ``train_valid_test_datasets_provider`` factory, +and the model's registry entry maps a ``--dataset-provider`` name to that +factory's dotted path. To add a new dataset (e.g. NLVR2): + +1. Create ``examples/multimodal_dev/data/.py`` with:: + + def train_valid_test_datasets_provider(train_val_test_num_samples): + ... # build datasets using args from get_args() + return train_ds, val_ds, test_ds + +2. Register it under the relevant model in + ``examples/multimodal_dev/models/__init__.py``:: + + MODEL_REGISTRY["qwen35_vl"]["dataset_providers"][""] = ( + "examples.multimodal_dev.data." + ".train_valid_test_datasets_provider" + ) + +3. Launch with ``--dataset-provider ``. + +No edits to ``pretrain_multimodal.py`` or ``forward_step.py`` are required. +""" + +import json +import logging +import random +from typing import Dict, List, Optional + +import torch +from torch.utils.data import Dataset + +try: + from qwen_vl_utils import process_vision_info + HAVE_QWEN_VL_UTILS = True +except ImportError: + HAVE_QWEN_VL_UTILS = False + +logger = logging.getLogger(__name__) + +# Qwen-VL recommended pixel-budget range; lets the processor pick a +# per-image patch grid that respects aspect ratio. +_QWEN_VL_MIN_PIXELS = 256 * 28 * 28 # 200_704 +_QWEN_VL_MAX_PIXELS = 1280 * 28 * 28 # 1_003_520 + + +# --------------------------------------------------------------------------- +# CORD-V2 helpers +# --------------------------------------------------------------------------- + +def _json2token(obj, sort_json_key=True): + """Convert a JSON object to a token-sequence string (Donut format).""" + if isinstance(obj, dict): + if len(obj) == 1 and "text_sequence" in obj: + return obj["text_sequence"] + output = "" + keys = sorted(obj.keys(), reverse=True) if sort_json_key else obj.keys() + for k in keys: + output += f"" + _json2token(obj[k], sort_json_key) + f"" + return output + if isinstance(obj, list): + return "".join(_json2token(item, sort_json_key) for item in obj) + return str(obj) + + +def load_cord_v2(split="train"): + """Load CORD-V2 and return a list of ``{image, question, answer}`` dicts.""" + from datasets import load_dataset + + ds = load_dataset("naver-clova-ix/cord-v2", split=split) + rng = random.Random(42) + examples = [] + for ex in ds: + gt = json.loads(ex["ground_truth"]) + gt_jsons = gt.get("gt_parses") or [gt["gt_parse"]] + text = rng.choice( + [_json2token(g, sort_json_key=True) for g in gt_jsons] + ) + examples.append( + {"image": ex["image"], "question": "Describe this image.", "answer": text} + ) + return examples + + +# --------------------------------------------------------------------------- +# Dataset +# --------------------------------------------------------------------------- + +class CordV2VLMDataset(Dataset): + """Single-turn VLM dataset backed by CORD-V2. + + Each sample is tokenized by the HF ``AutoProcessor`` and the image is + handed to the processor with Qwen-VL's dynamic-resolution budget + (``min_pixels`` / ``max_pixels``); the per-image patch grid varies with + aspect ratio. + + Args: + examples: Output of :func:`load_cord_v2`. + processor: ``AutoProcessor`` instance. + seq_length: End-truncate ``input_ids`` at this length. + image_token_id: Token ID for image placeholders. + target_length: Virtual dataset length (repeats examples if needed). + + NOTE: + For the Qwen3.5-VL processor, the temporal patch dimension is + always 2 (the processor duplicates a single frame so the 3D conv + behaves like a 2D conv on one image) — ``image_grid_thw`` therefore + has shape ``[num_images, 3]`` with ``T=2`` per image. + ``pixel_values`` has shape ``[total_patches, 3 * T * P * P]`` where + ``P`` is the processor's patch size. + """ + + def __init__( + self, + examples: List[Dict], + processor, + seq_length: int = 2048, + image_token_id: Optional[int] = None, + target_length: Optional[int] = None, + ): + if not HAVE_QWEN_VL_UTILS: + raise ImportError( + "qwen_vl_utils is required for Qwen3.5-VL preprocessing. " + "Install with `pip install qwen-vl-utils`.", + ) + self.examples = examples + self.processor = processor + self.seq_length = seq_length + self._length = target_length if target_length else len(examples) + tok = processor.tokenizer + # Falling back to 0 is unsafe: token 0 is a real vocab token in many + # tokenizers (incl. Qwen) and would be silently masked. Prefer EOS, + # and require at least one of pad/eos to be set. + if tok.pad_token_id is not None: + self.pad_token_id = int(tok.pad_token_id) + elif tok.eos_token_id is not None: + self.pad_token_id = int(tok.eos_token_id) + else: + raise ValueError( + "Tokenizer has neither pad_token_id nor eos_token_id; " + "cannot derive a safe pad id for loss masking.", + ) + + # Resolve image token ID. Vision embeddings are scattered into + # positions equal to this id by the model, so a wrong id silently + # breaks training — fail loudly rather than return None. + if image_token_id is not None: + self.image_token_id = int(image_token_id) + else: + vocab = tok.get_vocab() + for candidate in ("<|image_pad|>", "<|placeholder|>"): + if candidate in vocab: + self.image_token_id = int(vocab[candidate]) + break + else: + raise ValueError( + "Could not resolve image token id from tokenizer " + f"({type(tok).__name__}); pass --image-token-id " + "explicitly.", + ) + + # Structural tokens that must never appear as a loss target: + # pad, image, plus everything the tokenizer registered as special + # (im_start/im_end, vision_start/vision_end, video_pad, endoftext...). + # Mirrors megatron-bridge's extract_skipped_token_ids convention. + skipped: set = set(int(x) for x in (tok.all_special_ids or [])) + skipped.add(self.pad_token_id) + skipped.add(self.image_token_id) + self.skipped_token_ids = torch.tensor( + sorted(skipped), dtype=torch.long, + ) + + def __len__(self) -> int: + return self._length + + def _mark_assistant_span( + self, + input_ids_list: List[int], + asst_text: str, + loss_mask: torch.Tensor, + ) -> bool: + """Find ``asst_text`` as a contiguous token span in ``input_ids_list`` + and set ``loss_mask`` to 1 over those positions. + + Substring tokenization is sensitive to surrounding whitespace and + BPE merge boundaries, so we try a few common variants. Returns + True if a span was found. + """ + tokenizer = self.processor.tokenizer + n = len(input_ids_list) + variants = ( + asst_text, + asst_text + "\n", + asst_text.strip(), + asst_text.strip() + "\n", + ) + for variant in variants: + span_tokens = tokenizer( + variant, add_special_tokens=False, + )["input_ids"] + m = len(span_tokens) + if m == 0 or m > n: + continue + # Backward search: rightmost match = the actual assistant turn. + for start in range(n - m, -1, -1): + if input_ids_list[start : start + m] == span_tokens: + loss_mask[start : start + m] = 1.0 + return True + return False + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + example = self.examples[idx % len(self.examples)] + + # Conversation schema must include the actual image object inside + # the content so the chat template + process_vision_info can extract + # it (matches megatron-bridge's qwen2_5_collate_fn convention; also + # used by the Qwen3-VL processor). + conversation = [ + { + "role": "user", + "content": [ + {"type": "image", "image": example["image"]}, + {"type": "text", "text": example["question"]}, + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": example["answer"]}], + }, + ] + + text = self.processor.apply_chat_template( + conversation, tokenize=False, add_generation_prompt=False, + ) + images, _ = process_vision_info(conversation) + batch = self.processor( + text=[text], + images=images, + return_tensors="pt", + min_pixels=_QWEN_VL_MIN_PIXELS, + max_pixels=_QWEN_VL_MAX_PIXELS, + ) + + input_ids = batch["input_ids"].squeeze(0) + pixel_values = batch["pixel_values"].to(torch.bfloat16) + image_grid_thw = batch["image_grid_thw"] # [num_images, 3] + + # End-truncate so the model never sees more than seq_length tokens. + # Qwen-VL chat template puts the image at the user-turn start and the + # assistant answer trails at the end, so end-truncation preserves the + # image_pad block for normal-sized images; if the image grid alone + # already exceeds seq_length, the model will fail loudly at the + # masked_scatter step. + if input_ids.shape[0] > self.seq_length: + logger.warning( + "Sample idx=%d has %d tokens > seq_length=%d; truncating.", + idx, input_ids.shape[0], self.seq_length, + ) + input_ids = input_ids[: self.seq_length] + + # SFT loss mask: start fully masked, then unmask only the assistant + # answer span found via substring token search (mirrors + # megatron-bridge's create_multiturn_loss_mask_by_search). The user + # turn, chat-template tags, and image tokens stay masked. + loss_mask = torch.zeros_like(input_ids, dtype=torch.float32) + found = self._mark_assistant_span( + input_ids.tolist(), example["answer"], loss_mask, + ) + if not found: + logger.warning( + "Assistant span not located for example idx=%d; " + "loss_mask will be all-zero for this sample.", + idx, + ) + + # Shifted next-token labels: labels[i] is the target for position i. + labels = input_ids.clone() + labels[:-1] = input_ids[1:] + labels[-1] = -100 + + # Mask structural tokens on the *labels* (the prediction targets), + # not on input_ids — matches the next-token timeline. + labels[torch.isin(labels, self.skipped_token_ids)] = -100 + + # Shift loss_mask left by one so position i decides whether to learn + # input_ids[i] -> labels[i] (== input_ids[i+1]). Last position is + # never trained (no next token to predict). + loss_mask = torch.cat( + [loss_mask[1:], torch.zeros(1, dtype=loss_mask.dtype)], + ) + + # Enforce label = -100 wherever we won't compute loss. + labels[loss_mask == 0] = -100 + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "pixel_values": pixel_values, + "image_grid_thw": image_grid_thw, + } + + +# --------------------------------------------------------------------------- +# Megatron dataset provider interface +# --------------------------------------------------------------------------- + +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Provide CORD-V2 train / val / test datasets. + + Requires ``--hf-processor-path`` to point to a HuggingFace VL model + (e.g. ``Qwen/Qwen3.5-397B-A17B``) whose processor handles tokenization + and image preprocessing. + """ + from transformers import AutoProcessor + + from megatron.training import get_args + + args = get_args() + + processor_path = getattr(args, "hf_processor_path", None) + if processor_path is None: + raise ValueError( + "cord_v2 dataset requires --hf-processor-path " + "(e.g. Qwen/Qwen3.5-397B-A17B)" + ) + processor = AutoProcessor.from_pretrained( + processor_path, trust_remote_code=True, + ) + + seq_length = ( + getattr(args, "total_seq_length", None) + or getattr(args, "seq_length", 2048) + ) + image_token_id = getattr(args, "image_token_id", None) + + # Load real data + train_examples = load_cord_v2(split="train") + val_examples = load_cord_v2(split="validation") + test_examples = load_cord_v2(split="test") + + def _make(examples, num_samples): + return CordV2VLMDataset( + examples=examples, + processor=processor, + seq_length=seq_length, + image_token_id=image_token_id, + target_length=num_samples, + ) + + # MegatronPretrainingSampler asserts total_samples > 0, so val/test + # datasets must have non-zero length even when eval is disabled. + train_ds = _make(train_examples, train_val_test_num_samples[0]) + val_ds = _make(val_examples, max(train_val_test_num_samples[1], 1)) + test_ds = _make(test_examples, max(train_val_test_num_samples[2], 1)) + + return train_ds, val_ds, test_ds + + +if __name__ == "__main__": + from transformers import AutoProcessor + processor = AutoProcessor.from_pretrained( + "Qwen/Qwen3.5-397B-A17B", trust_remote_code=True, + ) + examples = load_cord_v2(split="train") + dataset = CordV2VLMDataset( + examples=examples, + processor=processor, + image_token_id=248056, + ) + print(dataset[0]) diff --git a/examples/multimodal_dev/data/mock.py b/examples/multimodal_dev/data/mock.py new file mode 100644 index 00000000000..0975b132013 --- /dev/null +++ b/examples/multimodal_dev/data/mock.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Mock dataset for multimodal_dev end-to-end testing. + +Generates synthetic image + text data. Each sample has random text +tokens with image-token placeholders, random pixel values sized for the +vision encoder, 3D MRoPE position IDs, and shifted labels. +""" + +import torch +from torch.utils.data import Dataset + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, +) +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index + + +class MockQwen35VLDataset(Dataset): + """Synthetic Qwen3.5-VL training samples. + + Args: + num_samples: Number of samples. + seq_length: Total sequence length (text + image tokens). + image_seq_length: Number of image tokens per sample. + vocab_size: Vocabulary size for random text tokens. + image_token_id: Token ID for image placeholders. + video_token_id: Token ID for video placeholders. + vision_start_token_id: Token ID marking start of a vision region. + image_size: Image height and width in pixels. + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + spatial_merge_size: Spatial merge factor. + """ + + def __init__( + self, + num_samples: int = 1000, + seq_length: int = 1024, + image_seq_length: int = 256, + vocab_size: int = 248320, + image_token_id: int = QWEN35_VL_IMAGE_TOKEN_ID, + video_token_id: int = QWEN35_VL_VIDEO_TOKEN_ID, + vision_start_token_id: int = QWEN35_VL_VISION_START_TOKEN_ID, + image_size: int = 224, + patch_size: int = 16, + temporal_patch_size: int = 2, + spatial_merge_size: int = 2, + ): + self.num_samples = num_samples + self.seq_length = seq_length + self.vocab_size = vocab_size + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.image_size = image_size + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.spatial_merge_size = spatial_merge_size + + h_patches = image_size // patch_size + w_patches = image_size // patch_size + t_patches = temporal_patch_size + self.grid_thw = torch.tensor([[t_patches, h_patches, w_patches]]) + + self.num_merged_tokens = ( + t_patches + * (h_patches // spatial_merge_size) + * (w_patches // spatial_merge_size) + ) + self.image_seq_length = min( + image_seq_length, self.num_merged_tokens, + ) + self.total_patches = t_patches * h_patches * w_patches + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + # Reserve 1 slot for the vision_start sentinel before image tokens. + text_length = self.seq_length - self.image_seq_length - 1 + text_tokens = torch.randint( + 1, self.vocab_size, (text_length,), dtype=torch.long, + ) + special_ids = { + self.image_token_id, + self.video_token_id, + self.vision_start_token_id, + } + for sid in special_ids: + text_tokens[text_tokens == sid] = 1 + + prefix_len = text_length // 2 + suffix_len = text_length - prefix_len + input_ids = torch.cat([ + text_tokens[:prefix_len], + torch.tensor( + [self.vision_start_token_id], dtype=torch.long, + ), + torch.full( + (self.image_seq_length,), + self.image_token_id, + dtype=torch.long, + ), + text_tokens[prefix_len: prefix_len + suffix_len], + ]) + + labels = input_ids.clone() + labels[:-1] = input_ids[1:] + labels[-1] = 0 + + loss_mask = (input_ids != self.image_token_id).float() + loss_mask[-1] = 0 + + pixel_dim = ( + 3 + * self.temporal_patch_size + * self.patch_size + * self.patch_size + ) + pixel_values = torch.randn(self.total_patches, pixel_dim) + + image_grid_thw = self.grid_thw.clone() + + position_ids, _ = get_rope_index( + spatial_merge_size=self.spatial_merge_size, + image_token_id=self.image_token_id, + video_token_id=self.video_token_id, + vision_start_token_id=self.vision_start_token_id, + input_ids=input_ids.unsqueeze(0), + image_grid_thw=image_grid_thw, + ) + position_ids = position_ids.squeeze(1) + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "cu_seqlens": torch.tensor([0, self.seq_length], dtype=torch.int32), + "cu_seqlens_padded": torch.tensor( + [0, self.seq_length], dtype=torch.int32, + ), + "max_seqlen": torch.tensor(self.seq_length, dtype=torch.int32), + "position_ids": position_ids, + "pixel_values": pixel_values, + "image_grid_thw": image_grid_thw, + } + + +def mock_collate_fn(batch): + """Collate: handles position_ids ``[3, S]`` stacking.""" + result = {} + keys = batch[0].keys() + for key in keys: + tensors = [sample[key] for sample in batch] + if key == "position_ids": + result[key] = torch.stack(tensors, dim=1) + elif key == "image_grid_thw": + result[key] = torch.cat(tensors, dim=0) + elif key == "pixel_values": + result[key] = torch.cat(tensors, dim=0) + else: + result[key] = torch.stack(tensors, dim=0) + return result + + +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Provide mock train / val / test datasets.""" + from megatron.training import get_args + + args = get_args() + kwargs = dict( + seq_length=getattr(args, "total_seq_length", 1024), + image_seq_length=getattr(args, "image_seq_length", 256), + vocab_size=getattr(args, "padded_vocab_size", 248320), + image_token_id=getattr(args, "image_token_id", 248056), + image_size=getattr(args, "image_size", 224), + ) + + train_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[0], **kwargs, + ) + val_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[1], **kwargs, + ) + test_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[2], **kwargs, + ) + + return train_ds, val_ds, test_ds diff --git a/examples/multimodal_dev/forward_step.py b/examples/multimodal_dev/forward_step.py new file mode 100644 index 00000000000..2729d4cf720 --- /dev/null +++ b/examples/multimodal_dev/forward_step.py @@ -0,0 +1,411 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Forward step, TP broadcast, and loss for multimodal_dev training.""" + +import math +from functools import partial +from itertools import accumulate +from typing import Any, Dict, Iterator, Optional + +import torch +import torch.nn.functional as F + +from megatron.core import mpu +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.parallel_state import ( + get_tensor_model_parallel_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_src_rank, +) +from megatron.training import get_args + +# ------------------------------------------------------------------- +# dtype <-> int mapping for cross-rank broadcast +# ------------------------------------------------------------------- + +_DTYPE_MAP = { + torch.float32: 0, + torch.float16: 1, + torch.bfloat16: 2, + torch.int64: 3, + torch.int32: 4, + torch.bool: 5, +} +_ID_MAP = {v: k for k, v in _DTYPE_MAP.items()} + + +def _dtype_to_id(dtype): + return _DTYPE_MAP.get(dtype, 0) + + +def _id_to_dtype(id_val): + return _ID_MAP.get(id_val, torch.float32) + + +# ------------------------------------------------------------------- +# Tensor broadcast helper +# ------------------------------------------------------------------- + + +def _broadcast_tensor(tensor, src, group, device): + """Broadcast a single tensor from *src* to all ranks in *group*.""" + ndim = torch.tensor( + [len(tensor.shape) if tensor is not None else 0], dtype=torch.long, device=device + ) + torch.distributed.broadcast(ndim, src, group=group) + + if ndim.item() == 0: + return None + + if tensor is not None: + shape_tensor = torch.tensor(list(tensor.shape), dtype=torch.long, device=device) + dtype_id = torch.tensor([_dtype_to_id(tensor.dtype)], dtype=torch.long, device=device) + else: + shape_tensor = torch.zeros(ndim.item(), dtype=torch.long, device=device) + dtype_id = torch.zeros(1, dtype=torch.long, device=device) + + torch.distributed.broadcast(shape_tensor, src, group=group) + torch.distributed.broadcast(dtype_id, src, group=group) + + dtype = _id_to_dtype(dtype_id.item()) + shape = tuple(shape_tensor.tolist()) + + if tensor is None: + tensor = torch.empty(shape, dtype=dtype, device=device) + torch.distributed.broadcast(tensor, src, group=group) + return tensor + + +# ------------------------------------------------------------------- +# Batch broadcast across TP ranks +# ------------------------------------------------------------------- + + +def broadcast_data_batch(data, device="cuda"): + """Broadcast a data-batch dict from TP rank 0 to all TP ranks.""" + src = get_tensor_model_parallel_src_rank() + group = get_tensor_model_parallel_group() + + if data is None: + data = {} + + if get_tensor_model_parallel_rank() == 0: + keys = list(data.keys()) + key_str = ",".join(keys) + key_bytes = key_str.encode("utf-8") + key_len = torch.tensor([len(key_bytes)], dtype=torch.long, device=device) + else: + key_len = torch.zeros(1, dtype=torch.long, device=device) + keys = [] + + torch.distributed.broadcast(key_len, src, group=group) + + if get_tensor_model_parallel_rank() == 0: + key_tensor = torch.tensor(list(key_bytes), dtype=torch.uint8, device=device) + else: + key_tensor = torch.zeros(key_len.item(), dtype=torch.uint8, device=device) + + torch.distributed.broadcast(key_tensor, src, group=group) + + if get_tensor_model_parallel_rank() != 0: + key_str = bytes(key_tensor.cpu().tolist()).decode("utf-8") + keys = key_str.split(",") if key_str else [] + + result = {} + for key in keys: + tensor = data.get(key, None) if data else None + if tensor is not None and isinstance(tensor, torch.Tensor): + tensor = tensor.to(device) + result[key] = _broadcast_tensor( + tensor if isinstance(tensor, torch.Tensor) else None, src, group, device + ) + + return result + + +# ------------------------------------------------------------------- +# THD (packed sequence) helpers +# ------------------------------------------------------------------- + + +def _build_packed_seq_params(seq_lengths: torch.Tensor, device: torch.device) -> PackedSeqParams: + """Build ``PackedSeqParams`` from per-sample valid sequence lengths. + + Args: + seq_lengths: ``[B]`` valid token counts per sample. + device: Target device for cu_seqlens tensors. + + Returns: + A ``PackedSeqParams`` instance with ``qkv_format='thd'``. + """ + if not isinstance(seq_lengths, torch.Tensor): + seq_lengths = torch.tensor(seq_lengths) + lengths_t = seq_lengths.to(device=device, dtype=torch.int32) + cu_seqlens = torch.zeros(lengths_t.numel() + 1, dtype=torch.int32, device=device) + torch.cumsum(lengths_t, dim=0, out=cu_seqlens[1:]) + max_seqlen = int(lengths_t.max().item()) + return _build_packed_seq_params_from_cu_seqlens(cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + + +def _build_packed_seq_params_from_cu_seqlens( + cu_seqlens: torch.Tensor, max_seqlen: int +) -> PackedSeqParams: + """Build ``PackedSeqParams`` from packed cumulative sequence lengths. + + ``cu_seqlens`` must already be on the target compute device. + """ + cs = cu_seqlens.to(dtype=torch.int32) + total_tokens = int(cs[-1].item()) + return PackedSeqParams( + cu_seqlens_q=cs, + cu_seqlens_kv=cs, + cu_seqlens_q_padded=cs, + cu_seqlens_kv_padded=cs, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + qkv_format='thd', + total_tokens=total_tokens, + ) + + +def pack_or_pad_batch( + batch: Optional[list[Dict[str, Any]]], + use_packed_sequence: bool = False, + seq_length: Optional[int] = None, + device="cuda", +) -> Dict[str, Any]: + """Pack or pad a ``[B, S]`` batch into ``[1, T]`` THD or ``[B, S]`` BSHD. + + Must be invoked on every TP rank. On the TP source rank ``batch`` is + the per-sample dict list from the dataset; on other TP ranks ``batch`` + may be ``None`` (the function relies on the trailing TP broadcast to + distribute results). All metadata needed to reconstruct + ``PackedSeqParams`` (``cu_seqlens``, ``cu_seqlens_padded``, + ``max_seqlen``, ``total_tokens``) is broadcast alongside the data, so + every rank can build an identical ``PackedSeqParams`` on its own. + """ + tp_size = mpu.get_tensor_model_parallel_world_size() + cp_size = mpu.get_context_parallel_world_size() + is_src = mpu.get_tensor_model_parallel_rank() == 0 + + # SP is an explicit runtime option; TP>1 does not imply SP is enabled. + # get_args() itself raises in test contexts where megatron globals are + # not initialised. + try: + has_sp = bool(getattr(get_args(), "sequence_parallel", False)) + except AssertionError: + has_sp = False + + if cp_size > 1: + divisible_by = (tp_size * cp_size * 2) if has_sp else (cp_size * 2) + else: + divisible_by = tp_size if has_sp else 1 + + if use_packed_sequence: + packed_batch: Dict[str, Any] = {} + + if is_src: + assert batch is not None, "source TP rank must provide a batch" + input_ids_list, labels_list, loss_mask_list = [], [], [] + pixel_values_list, image_grid_thw_list = [], [] + seqlens_list, seqlens_padded_list = [], [] + + for sample in batch: + seqlen = sample["input_ids"].shape[0] + assert ( + sample["labels"].shape == sample["input_ids"].shape == sample["loss_mask"].shape + ), "labels, input_ids, and loss_mask must have the same shape" + target_len = math.ceil(seqlen / divisible_by) * divisible_by + input_ids_list.append(F.pad(sample["input_ids"], (0, target_len - seqlen), value=0)) + labels_list.append(F.pad(sample["labels"], (0, target_len - seqlen), value=-100)) + loss_mask_list.append(F.pad(sample["loss_mask"], (0, target_len - seqlen), value=0)) + seqlens_list.append(seqlen) + seqlens_padded_list.append(target_len) + pixel_values_list.append(sample["pixel_values"]) + image_grid_thw_list.append(sample["image_grid_thw"]) + + cu_seqlens = list(accumulate(seqlens_list, initial=0)) + cu_seqlens_padded = list(accumulate(seqlens_padded_list, initial=0)) + + packed_batch["input_ids"] = torch.concat(input_ids_list, dim=0).unsqueeze(0) + packed_batch["labels"] = torch.concat(labels_list, dim=0).unsqueeze(0) + packed_batch["loss_mask"] = torch.concat(loss_mask_list, dim=0).unsqueeze(0) + packed_batch["pixel_values"] = torch.concat(pixel_values_list) + packed_batch["image_grid_thw"] = torch.concat(image_grid_thw_list) + # cu_seqlens / cu_seqlens_padded need to reach non-source TP ranks + # so each rank can build an identical PackedSeqParams. + packed_batch["cu_seqlens"] = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) + packed_batch["cu_seqlens_padded"] = torch.tensor( + cu_seqlens_padded, dtype=torch.int32, device=device + ) + + packed_batch = broadcast_data_batch(packed_batch, device=device) + + cu_seqlens_t = packed_batch.pop("cu_seqlens") + cu_seqlens_padded_t = packed_batch.pop("cu_seqlens_padded") + # Derive max_seqlen / total_tokens from the (broadcast) cu_seqlens — + # no extra collective needed. + max_seqlen_q = int((cu_seqlens_padded_t[1:] - cu_seqlens_padded_t[:-1]).max().item()) + total_tokens = int(cu_seqlens_padded_t[-1].item()) + + packed_batch["packed_seq_params"] = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_t, + cu_seqlens_kv=cu_seqlens_t, + cu_seqlens_q_padded=cu_seqlens_padded_t, + cu_seqlens_kv_padded=cu_seqlens_padded_t, + max_seqlen_q=max_seqlen_q, + max_seqlen_kv=max_seqlen_q, + total_tokens=total_tokens, + ) + return packed_batch + + # ---------- padded (BSHD) branch ---------- + assert seq_length is not None, "seq_length must be provided when use_packed_sequence is False" + padded_batch: Dict[str, Any] = {} + + if is_src: + assert batch is not None, "source TP rank must provide a batch" + max_seqlens = max(x["input_ids"].shape[0] for x in batch) + target_seqlens = min(max_seqlens, seq_length) + # Round target seqlen up to the parallelism alignment factor so the + # batched tensor is divisible for CP (+SP) splitting downstream. + if divisible_by > 1: + target_seqlens = math.ceil(target_seqlens / divisible_by) * divisible_by + + for sample in batch: + sample["input_ids"] = F.pad( + sample["input_ids"], (0, target_seqlens - sample["input_ids"].shape[0]), value=0 + ) + sample["labels"] = F.pad( + sample["labels"], (0, target_seqlens - sample["labels"].shape[0]), value=-100 + ) + sample["loss_mask"] = F.pad( + sample["loss_mask"], (0, target_seqlens - sample["loss_mask"].shape[0]), value=0 + ) + + padded_batch["input_ids"] = torch.concat( + [x["input_ids"].unsqueeze(0) for x in batch], dim=0 + ) + padded_batch["labels"] = torch.concat([x["labels"].unsqueeze(0) for x in batch], dim=0) + padded_batch["loss_mask"] = torch.concat( + [x["loss_mask"].unsqueeze(0) for x in batch], dim=0 + ) + padded_batch["pixel_values"] = torch.concat([x["pixel_values"] for x in batch]) + padded_batch["image_grid_thw"] = torch.concat([x["image_grid_thw"] for x in batch]) + + return broadcast_data_batch(padded_batch, device=device) + + +# ------------------------------------------------------------------- +# get_batch +# ------------------------------------------------------------------- + + +def get_batch(data_iterator: Iterator[list[Dict[str, Any]]]): + """Get a batch from *data_iterator* and broadcast across TP ranks.""" + device = "cuda" + args = get_args() + + if get_tensor_model_parallel_rank() == 0: + try: + data = next(data_iterator) + has_data = torch.tensor([1], dtype=torch.uint8, device=device) + except StopIteration: + has_data = torch.tensor([0], dtype=torch.uint8, device=device) + data = None + else: + has_data = torch.empty(1, dtype=torch.uint8, device=device) + data = None + + src = get_tensor_model_parallel_src_rank() + group = get_tensor_model_parallel_group() + torch.distributed.broadcast(has_data, src, group=group) + + if has_data.item() == 0: + return None + + # Because broadcast will not broadcast packed_seq_params, we move it into pack_or_pad_batch + batch = pack_or_pad_batch(data, args.use_packed_sequence, args.seq_length, device=device) + + # Fix shapes produced by default_collate. + if "position_ids" in batch and batch["position_ids"] is not None: + p = batch["position_ids"] + if p.dim() == 3 and p.shape[1] == 3: + batch["position_ids"] = p.permute(1, 0, 2).contiguous() + + if "pixel_values" in batch and batch["pixel_values"] is not None: + pv = batch["pixel_values"] + if pv.dim() == 3: + B, P, D = pv.shape + batch["pixel_values"] = pv.reshape(B * P, D) + + if "image_grid_thw" in batch and batch["image_grid_thw"] is not None: + g = batch["image_grid_thw"] + if g.dim() == 3: + batch["image_grid_thw"] = g.squeeze(1) + + return batch + + +# ------------------------------------------------------------------- +# Loss +# ------------------------------------------------------------------- + + +def loss_func(loss_mask, output_tensor): + """Compute masked language model loss.""" + losses = output_tensor.float() + loss_mask = loss_mask.contiguous().view(-1).float() + + total_tokens = loss_mask.sum().clone().detach().to(torch.int) + total_loss = torch.sum(losses.view(-1) * loss_mask) + reporting_loss = torch.cat([total_loss.clone().detach().view(1), total_tokens.view(1)]) + + return (total_loss, total_tokens, {"lm loss": reporting_loss}) + + +# ------------------------------------------------------------------- +# Forward step +# ------------------------------------------------------------------- + + +def forward_step(data_iterator, model): + """Forward step for multimodal_dev training.""" + batch = get_batch(data_iterator) + + if batch is None: + return None, None + + pixel_values = batch.get("pixel_values", None) + if ( + pixel_values is not None + and pixel_values.is_floating_point() + and pixel_values.dtype == torch.float32 + ): + pixel_values = pixel_values.bfloat16() + + # We don't provide position_ids, now. Let model handle it itself. + output_tensor = model( + input_ids=batch["input_ids"], + position_ids=batch.get("position_ids"), + attention_mask=batch.get("attention_mask", None), + labels=batch.get("labels", None), + loss_mask=batch.get("loss_mask", None), + pixel_values=pixel_values, + image_grid_thw=batch.get("image_grid_thw", None), + packed_seq_params=batch.get("packed_seq_params", None), + ) + + loss_mask = batch.get("loss_mask", None) + if loss_mask is None: + loss_mask = torch.ones_like(batch["input_ids"], dtype=torch.float) + + # Slice loss_mask the same way the model sliced its inputs, so the + # mask aligns with the CP-shard output. Delegated to MultimodalModel + # so the slicing rule lives in one place. + from examples.multimodal_dev.models.base import MultimodalModel + + loss_mask = MultimodalModel.cp_split_loss_mask(loss_mask, batch.get("packed_seq_params", None)) + + return output_tensor, partial(loss_func, loss_mask) diff --git a/examples/multimodal_dev/models/__init__.py b/examples/multimodal_dev/models/__init__.py new file mode 100644 index 00000000000..225414055e6 --- /dev/null +++ b/examples/multimodal_dev/models/__init__.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Model registry for multimodal_dev training. + +Maps ``--model-arch`` to a set of factory functions that fully encapsulate +model-specific logic. The training entry point (``pretrain_multimodal.py``) +remains model-agnostic — adding a new architecture only requires a new +registry entry (and its backing module) without touching the entry point. + +Registry entry fields +--------------------- +``model_factory_fn`` *(required)* + ``(args, language_config, vision_config, **kwargs) -> MegatronModule`` + Builds and returns the complete model instance. + +``vision_config_fn`` *(required)* + ``(num_layers_override=None, variant=None) -> TransformerConfig`` + Returns the vision encoder TransformerConfig. + +``post_language_config_fn`` *(optional)* + ``(language_config, args) -> None`` + Mutates the language TransformerConfig in-place with model-specific + fields (e.g. ``mrope_section``). + +``vision_flops_fn`` *(optional)* + ``(args, language_config, vision_config) -> None`` + Sets vision FLOPs metadata on ``args`` for training throughput logging. + +``dataset_providers`` *(optional)* + ``Dict[str, str | callable]`` + Maps ``--dataset-provider`` names to callables (or dotted import paths + resolved lazily) with signature + ``(train_val_test_num_samples) -> (train_ds, val_ds, test_ds)``. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import get_qwen35_vl_vision_config +from examples.multimodal_dev.models.qwen35_vl.factory import build_model as _build_qwen35_vl_model +from examples.multimodal_dev.models.qwen35_vl.factory import ( + post_language_config as _qwen35_vl_post_language_config, +) +from examples.multimodal_dev.models.qwen35_vl.factory import ( + set_vision_flops_metadata as _qwen35_vl_vision_flops, +) + +MODEL_REGISTRY = { + "qwen35_vl": { + "model_factory_fn": _build_qwen35_vl_model, + "vision_config_fn": get_qwen35_vl_vision_config, + "post_language_config_fn": _qwen35_vl_post_language_config, + "vision_flops_fn": _qwen35_vl_vision_flops, + "dataset_providers": { + "mock": ( + "examples.multimodal_dev.data.mock" + ".train_valid_test_datasets_provider" + ), + "cord_v2": ( + "examples.multimodal_dev.data.cord_v2" + ".train_valid_test_datasets_provider" + ), + }, + }, +} diff --git a/examples/multimodal_dev/models/base.py b/examples/multimodal_dev/models/base.py new file mode 100644 index 00000000000..00c3b10a740 --- /dev/null +++ b/examples/multimodal_dev/models/base.py @@ -0,0 +1,369 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Base multimodal model for FSDP + EP training. + +Composes a vision encoder and a ``GPTModel`` language decoder. Designed +for FSDP + EP: always builds the **full** model on every rank (no PP +flags). PP support is only available through the MIMO ``MimoModel`` +assembly path. + +Subclasses override ``compute_position_ids()`` for model-specific +position encoding (e.g. MRoPE for Qwen3.5-VL). +""" + +import contextlib +from typing import Optional + +import torch +from torch import Tensor + +from megatron.core import parallel_state, tensor_parallel +from megatron.core.models.gpt import GPTModel +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + + +def _cp_split_tensor(tensor, seq_dim, cp_size, cp_rank): + """Zigzag-split *tensor* along *seq_dim* for context parallelism (BSHD). + + Splits the sequence into ``2 * cp_size`` equal chunks, then selects + chunks ``[cp_rank, 2*cp_size - cp_rank - 1]`` and concatenates them. + This mirrors ``megatron.core.utils.get_batch_on_this_cp_rank``. + """ + S = tensor.shape[seq_dim] + assert S % (2 * cp_size) == 0, f"seq_len {S} not divisible by 2*cp_size={2 * cp_size}" + tensor = tensor.view( + *tensor.shape[:seq_dim], 2 * cp_size, S // (2 * cp_size), *tensor.shape[seq_dim + 1 :] + ) + index = torch.zeros(2, dtype=torch.int64, device=tensor.device) + index[0] = cp_rank + index[1] = 2 * cp_size - cp_rank - 1 + tensor = tensor.index_select(seq_dim, index) + tensor = tensor.view(*tensor.shape[:seq_dim], -1, *tensor.shape[seq_dim + 2 :]) + return tensor + + +class _NoCPGroup: + """Dummy size-1 process group used to bypass MRoPE's BSHD-style + zigzag of pre-computed THD freqs (Megatron-Core gap: + ``MultimodalRotaryEmbedding.forward`` lacks the ``not packed_seq`` + skip that plain ``RotaryEmbedding`` has). + """ + + def size(self): + """Pretend this group has exactly one rank.""" + return 1 + + def rank(self): + """This rank's id within the fake group is always 0.""" + return 0 + + +_NO_CP_GROUP = _NoCPGroup() + +# Note: reported ``mtp_1 loss`` drifts ~1.3% from the CP=1 baseline under +# THD+CP. Megatron-Core's logging averages per-rank pre-divided ratios +# with op=AVG, and per-rank num_tokens are unequal after MTP rolling. +# Gradients are correct; only the *logged* value drifts. + + +def _thd_cp_partition_index(cu_seqlens_padded, total_tokens, cp_size, cp_rank): + """Per-rank token index for THD + CP via TE's + ``thd_get_partitioned_indices``. Cast to int64 so the result can be + used directly with ``index_select`` regardless of TE's return dtype. + """ + from transformer_engine.pytorch import cpp_extensions as tex + + idx = tex.thd_get_partitioned_indices(cu_seqlens_padded, total_tokens, cp_size, cp_rank) + return idx.long() + + +class MultimodalModel(MegatronModule): + """Base class for multimodal vision-language models. + + Composes a pre-constructed vision encoder and a ``GPTModel`` language + decoder. Designed for FSDP + EP; always builds the full model on + every rank. + + Args: + language_config: ``TransformerConfig`` for the language decoder. + language_spec: ``ModuleSpec`` for decoder transformer layers. + vision_encoder: Pre-constructed vision encoder module. + vocab_size: Language model vocabulary size. + max_sequence_length: Maximum sequence length. + image_token_id: Token ID for image placeholder tokens. + position_embedding_type: Position embedding type for the decoder. + rotary_percent: Fraction of hidden dim for RoPE. + rotary_base: Base frequency for RoPE. + mrope_section: MRoPE channel sections. + mtp_block_spec: Optional MTP block spec. + parallel_output: Keep outputs split across TP ranks. + share_embeddings_and_output_weights: Tie input/output embeddings. + """ + + def __init__( + self, + language_config: TransformerConfig, + language_spec: ModuleSpec, + vision_encoder: MegatronModule, + vocab_size: int, + max_sequence_length: int, + image_token_id: int, + position_embedding_type: str = "rope", + rotary_percent: float = 1.0, + rotary_base: int = 10000, + mrope_section: Optional[list] = None, + mtp_block_spec: Optional[ModuleSpec] = None, + parallel_output: bool = True, + share_embeddings_and_output_weights: bool = False, + ): + super().__init__(config=language_config) + + self.image_token_id = image_token_id + + self.vision_model = vision_encoder + self.language_model = GPTModel( + config=language_config, + transformer_layer_spec=language_spec, + vocab_size=vocab_size, + max_sequence_length=max_sequence_length, + pre_process=True, + post_process=True, + parallel_output=parallel_output, + share_embeddings_and_output_weights=(share_embeddings_and_output_weights), + position_embedding_type=position_embedding_type, + rotary_percent=rotary_percent, + rotary_base=rotary_base, + mtp_block_spec=mtp_block_spec, + ) + + def set_input_tensor(self, input_tensor): + """Route input tensors (simplified, no PP routing).""" + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + assert len(input_tensor) == 1 + self.language_model.set_input_tensor(input_tensor[0]) + + def _scatter_vision_embeddings( + self, input_ids: Tensor, text_embeddings: Tensor, vision_embeddings: Tensor + ) -> Tensor: + """Replace image-token positions with vision embeddings. + + Handles sequence parallelism (gather → scatter → re-scatter). + + Args: + input_ids: ``[B, S]`` token IDs. + text_embeddings: ``[S, B, D]`` (or ``[S/TP, B, D]`` with SP). + vision_embeddings: ``[num_visual_tokens, D]``. + + Returns: + Combined embeddings, same shape as *text_embeddings*. + """ + sp = ( + self.config.sequence_parallel + and parallel_state.get_tensor_model_parallel_world_size() > 1 + ) + + if sp: + text_embeddings = tensor_parallel.gather_from_sequence_parallel_region( + text_embeddings, tensor_parallel_output_grad=False + ) + + combined = text_embeddings.transpose(0, 1).contiguous() + image_mask = input_ids == self.image_token_id + mask_expanded = image_mask.unsqueeze(-1).expand_as(combined) + combined = combined.masked_scatter(mask_expanded, vision_embeddings) + combined = combined.transpose(0, 1).contiguous() + + if sp: + combined = tensor_parallel.scatter_to_sequence_parallel_region(combined) + + return combined + + def compute_position_ids( + self, input_ids: Tensor, image_grid_thw: Optional[Tensor] = None, packed_seq_params=None + ) -> Tensor: + """Compute position IDs. Override for MRoPE etc. + + Default: simple sequential positions. ``packed_seq_params`` is + accepted for subclass compatibility (e.g. MRoPE in THD mode). + """ + B, S = input_ids.shape + return torch.arange(S, device=input_ids.device).unsqueeze(0).expand(B, -1) + + def _cp_split_for_forward( + self, + *, + decoder_input, + input_ids, + labels, + loss_mask, + attention_mask, + position_ids, + packed_seq_params, + ): + """Apply CP split to model-forward inputs. + + BSHD path zigzag-splits each tensor along its seq dim. THD path + partitions per-sample via ``tex.thd_get_partitioned_indices`` so + chunks line up with ``cu_seqlens_q_padded`` boundaries. + ``position_ids`` and ``attention_mask`` are NOT split in THD — + MRoPE returns full freqs and TE attention's + ``_apply_rotary_pos_emb_thd`` does the per-sample CP zigzag + itself via ``_get_thd_freqs_on_this_cp_rank``. + """ + cp_size = parallel_state.get_context_parallel_world_size() + if cp_size <= 1: + return (decoder_input, input_ids, labels, loss_mask, attention_mask, position_ids) + cp_rank = parallel_state.get_context_parallel_rank() + + if packed_seq_params is not None: + total_tokens = ( + decoder_input.shape[0] if decoder_input is not None else input_ids.shape[1] + ) + idx = _thd_cp_partition_index( + packed_seq_params.cu_seqlens_q_padded, total_tokens, cp_size, cp_rank + ) + if decoder_input is not None: + decoder_input = decoder_input.index_select(0, idx) + if input_ids is not None: + input_ids = input_ids.index_select(1, idx) + if labels is not None: + labels = labels.index_select(1, idx) + if loss_mask is not None: + loss_mask = loss_mask.index_select(1, idx) + else: + + def _split(t, seq_dim): + return ( + None + if t is None + else _cp_split_tensor(t, seq_dim=seq_dim, cp_size=cp_size, cp_rank=cp_rank) + ) + + decoder_input = _split(decoder_input, 0) + input_ids = _split(input_ids, 1) + labels = _split(labels, 1) + loss_mask = _split(loss_mask, 1) + attention_mask = _split(attention_mask, 1) + + return (decoder_input, input_ids, labels, loss_mask, attention_mask, position_ids) + + @staticmethod + def cp_split_loss_mask(loss_mask, packed_seq_params): + """Slice ``loss_mask`` the same way the model slices its inputs. + + Mirrors the slicing done inside :meth:`_cp_split_for_forward` so + the loss computation outside the model can index a mask aligned + with the model's CP-shard output. Returns ``loss_mask`` unchanged + when ``CP <= 1``. + """ + cp_size = parallel_state.get_context_parallel_world_size() + if cp_size <= 1 or loss_mask is None: + return loss_mask + cp_rank = parallel_state.get_context_parallel_rank() + if packed_seq_params is not None: + idx = _thd_cp_partition_index( + packed_seq_params.cu_seqlens_q_padded, loss_mask.shape[1], cp_size, cp_rank + ) + return loss_mask.index_select(1, idx) + return _cp_split_tensor(loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + + @contextlib.contextmanager + def _thd_mrope_no_cp_override(self, packed_seq_params): + """Force ``rotary_pos_emb.cp_group`` to size 1 for the wrapped + forward call so MRoPE returns full-length freqs in THD mode. + Attention then applies per-sample CP zigzag itself via + ``_apply_rotary_pos_emb_thd``. Done by direct mutation rather + than via ``packed_seq_params.cp_group`` so MTP's CP-aware roll + (which reads that field) still sees the real CP group. + """ + mrope = ( + getattr(self.language_model, "rotary_pos_emb", None) + if packed_seq_params is not None + and parallel_state.get_context_parallel_world_size() > 1 + else None + ) + saved = getattr(mrope, "cp_group", None) if mrope is not None else None + if mrope is not None: + mrope.cp_group = _NO_CP_GROUP + try: + yield + finally: + if mrope is not None: + mrope.cp_group = saved + + def forward( + self, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor = None, + labels: Tensor = None, + loss_mask: Tensor = None, + pixel_values: Tensor = None, + image_grid_thw: Tensor = None, + decoder_input: Tensor = None, + packed_seq_params=None, + **kwargs, + ): + """Forward pass. + + Args: + input_ids: ``[B, S]`` token IDs (or ``[1, T]`` in THD mode). + position_ids: ``[3, B, S]`` for MRoPE or ``[B, S]`` + (``[3, 1, T]`` / ``[1, T]`` in THD mode). + attention_mask: ``[B, S]`` attention mask (None in THD). + labels: ``[B, S]`` target token IDs (``[1, T]`` in THD). + loss_mask: ``[B, S]`` mask for loss (``[1, T]`` in THD). + pixel_values: Preprocessed image pixels. + image_grid_thw: ``[num_images, 3]`` grid dimensions. + decoder_input: Pre-computed decoder input (skip embed). + packed_seq_params: ``PackedSeqParams`` for THD attention. + + Returns: + Loss tensor (post_process=True) or hidden states. + """ + if position_ids is None: + position_ids = self.compute_position_ids( + input_ids=input_ids, + image_grid_thw=image_grid_thw, + packed_seq_params=packed_seq_params, + ) + + vision_embeddings = None + if self.vision_model is not None and pixel_values is not None: + vision_embeddings = self.vision_model(pixel_values, image_grid_thw) + + if decoder_input is None and self.language_model is not None: + text_embeddings = self.language_model.embedding(input_ids=input_ids, position_ids=None) + + if vision_embeddings is not None: + decoder_input = self._scatter_vision_embeddings( + input_ids, text_embeddings, vision_embeddings + ) + else: + decoder_input = text_embeddings + + (decoder_input, input_ids, labels, loss_mask, attention_mask, position_ids) = ( + self._cp_split_for_forward( + decoder_input=decoder_input, + input_ids=input_ids, + labels=labels, + loss_mask=loss_mask, + attention_mask=attention_mask, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + ) + ) + + with self._thd_mrope_no_cp_override(packed_seq_params): + return self.language_model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + decoder_input=decoder_input, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + ) diff --git a/examples/multimodal_dev/models/qwen35_vl/__init__.py b/examples/multimodal_dev/models/qwen35_vl/__init__.py new file mode 100644 index 00000000000..1a0bad8b219 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/__init__.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Qwen3.5-VL model components — the single source of truth. + +Both the standalone ``multimodal_dev`` training path and the MIMO path +import from here. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + MROPE_SECTION, + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_END_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, + QWEN35_VL_VOCAB_SIZE, + ROTARY_BASE, + ROTARY_PERCENT, + VISION_KWARGS, + get_qwen35_vl_language_config, + get_qwen35_vl_vision_config, +) +from examples.multimodal_dev.models.qwen35_vl.factory import ( + build_model, + post_language_config, + set_vision_flops_metadata, +) +from examples.multimodal_dev.models.qwen35_vl.model import Qwen35VLModel +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index +from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_language_spec, + get_qwen35_vl_vision_spec, +) +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import ( + Qwen35VLPatchEmbed, + Qwen35VLPatchMerger, + Qwen35VLVisionEncoder, + Qwen35VLVisionRotaryEmbedding, +) + +__all__ = [ + # Model class + "Qwen35VLModel", + # Factory functions + "build_model", + "post_language_config", + "set_vision_flops_metadata", + # Vision encoder + "Qwen35VLVisionEncoder", + "Qwen35VLPatchEmbed", + "Qwen35VLPatchMerger", + "Qwen35VLVisionRotaryEmbedding", + # Config helpers + "get_qwen35_vl_vision_config", + "get_qwen35_vl_language_config", + # Spec helpers + "get_qwen35_vl_language_spec", + "get_qwen35_vl_vision_spec", + # MRoPE + "get_rope_index", + # Constants + "QWEN35_VL_IMAGE_TOKEN_ID", + "QWEN35_VL_VIDEO_TOKEN_ID", + "QWEN35_VL_VISION_START_TOKEN_ID", + "QWEN35_VL_VISION_END_TOKEN_ID", + "QWEN35_VL_VOCAB_SIZE", + "ROTARY_BASE", + "ROTARY_PERCENT", + "MROPE_SECTION", + "VISION_KWARGS", +] diff --git a/examples/multimodal_dev/models/qwen35_vl/configuration.py b/examples/multimodal_dev/models/qwen35_vl/configuration.py new file mode 100644 index 00000000000..81f73148314 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/configuration.py @@ -0,0 +1,355 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Configuration helpers for Qwen3.5-VL vision-language model. + +Provides TransformerConfig builders for the vision encoder and all language +decoder variants. Both the standalone ``multimodal_dev`` training path and the +MIMO path import from here — this is the single source of truth. + +Supported language variants (HuggingFace Qwen3.5 series): + ``0.8b`` Dense 0.8B + ``2b`` Dense 2B + ``4b`` Dense 4B + ``9b`` Dense 9B + ``27b`` Dense 27B + ``35b_a3b`` MoE 35B-A3B (256 experts, top-8) + ``122b_a10b`` MoE 122B-A10B (256 experts, top-8) + ``397b_a17b`` MoE 397B-A17B (512 experts, top-10) + ``35b_a3b_light`` Reduced 35B-A3B for testing + ``proxy`` Reduced proxy based on 397B for single-node testing +""" + +from typing import Optional + +import torch + +from megatron.core.transformer.transformer_config import TransformerConfig + +# --------------------------------------------------------------------------- +# Public constants +# --------------------------------------------------------------------------- + +QWEN35_VL_IMAGE_TOKEN_ID: int = 248056 +QWEN35_VL_VIDEO_TOKEN_ID: int = 248057 +QWEN35_VL_VISION_START_TOKEN_ID: int = 248053 +QWEN35_VL_VISION_END_TOKEN_ID: int = 248054 +QWEN35_VL_VOCAB_SIZE: int = 248320 + +ROTARY_BASE: int = 10_000_000 +ROTARY_PERCENT: float = 0.25 +MROPE_SECTION: list = [11, 11, 10] + +# --------------------------------------------------------------------------- +# Vision config +# --------------------------------------------------------------------------- + +VISION_KWARGS = { + "in_channels": 3, + "patch_size": 16, + "temporal_patch_size": 2, + "spatial_merge_size": 2, + "out_hidden_size": 3584, + "max_num_positions": 2304, +} + +# Three distinct vision encoder architectures in the Qwen3.5 family. +_VISION_SMALL = { + "num_layers": 12, "hidden_size": 768, "num_attention_heads": 12, + "kv_channels": 64, "ffn_hidden_size": 3072, +} +_VISION_MEDIUM = { + "num_layers": 24, "hidden_size": 1024, "num_attention_heads": 16, + "kv_channels": 64, "ffn_hidden_size": 4096, +} +_VISION_LARGE = { + "num_layers": 27, "hidden_size": 1152, "num_attention_heads": 16, + "kv_channels": 72, "ffn_hidden_size": 4304, +} + +# Per-variant vision config. ``out_hidden_size`` equals the language model's +# hidden_size and controls the merger projection output dimension. +_VISION_VARIANT_CONFIGS = { + "0.8b": {**_VISION_SMALL, "out_hidden_size": 1024}, + "2b": {**_VISION_MEDIUM, "out_hidden_size": 2048}, + "4b": {**_VISION_MEDIUM, "out_hidden_size": 2560}, + "9b": {**_VISION_LARGE, "out_hidden_size": 4096}, + "27b": {**_VISION_LARGE, "out_hidden_size": 5120}, + "35b_a3b": {**_VISION_LARGE, "out_hidden_size": 2048}, + "122b_a10b": {**_VISION_LARGE, "out_hidden_size": 3072}, + "397b_a17b": {**_VISION_LARGE, "out_hidden_size": 4096}, +} + +# Fallback for proxy/unknown variants (large ViT, generic out_hidden_size). +_VISION_DEFAULT = {**_VISION_LARGE, "out_hidden_size": 3584} + + +def get_qwen35_vl_vision_config( + num_layers_override: Optional[int] = None, + variant: Optional[str] = None, +) -> TransformerConfig: + """TransformerConfig for the Qwen3.5-VL vision encoder. + + Three ViT architectures are used across the family: + - Small (0.8b): depth 12, 768-dim, 12 heads + - Medium (2b, 4b): depth 24, 1024-dim, 16 heads + - Large (9b, 27b, MoE variants): depth 27, 1152-dim, 16 heads + + Args: + num_layers_override: Override vision backbone depth for proxy runs. + variant: Language model variant name. When set, selects the + matching vision config from ``_VISION_VARIANT_CONFIGS`` if one + exists; otherwise the default large-ViT config is used. + """ + vcfg = _VISION_VARIANT_CONFIGS.get(variant, _VISION_DEFAULT) + num_layers = vcfg["num_layers"] + if num_layers_override is not None: + num_layers = num_layers_override + + return TransformerConfig( + num_layers=num_layers, + hidden_size=vcfg["hidden_size"], + num_attention_heads=vcfg["num_attention_heads"], + kv_channels=vcfg["kv_channels"], + ffn_hidden_size=vcfg["ffn_hidden_size"], + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-6, + normalization="LayerNorm", + gated_linear_unit=False, + activation_func=lambda x: torch.nn.functional.gelu(x, approximate="tanh"), + bias_activation_fusion=False, + apply_query_key_layer_scaling=False, + apply_rope_fusion=False, + bf16=False, + ) + + +# --------------------------------------------------------------------------- +# Language config variants +# --------------------------------------------------------------------------- + +_VARIANT_CONFIGS = { + "0.8b": { + "num_layers": 24, + "hidden_size": 1024, + "ffn_hidden_size": 3584, + "num_attention_heads": 8, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 16, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "2b": { + "num_layers": 24, + "hidden_size": 2048, + "ffn_hidden_size": 6144, + "num_attention_heads": 8, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 16, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "4b": { + "num_layers": 32, + "hidden_size": 2560, + "ffn_hidden_size": 9216, + "num_attention_heads": 16, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "9b": { + "num_layers": 32, + "hidden_size": 4096, + "ffn_hidden_size": 12288, + "num_attention_heads": 16, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "27b": { + "num_layers": 64, + "hidden_size": 5120, + "ffn_hidden_size": 17408, + "num_attention_heads": 24, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 48, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "35b_a3b": { + "num_layers": 40, + "hidden_size": 2048, + "ffn_hidden_size": 4096, + "num_attention_heads": 16, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 512, + "moe_shared_expert_intermediate_size": 512, + }, + "35b_a3b_light": { + "num_layers": 20, + "hidden_size": 2048, + "ffn_hidden_size": 4096, + "num_attention_heads": 16, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 512, + "moe_shared_expert_intermediate_size": 512, + }, + "122b_a10b": { + "num_layers": 48, + "hidden_size": 3072, + "ffn_hidden_size": 8192, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, + "397b_a17b": { + "num_layers": 60, + "hidden_size": 4096, + "ffn_hidden_size": 10240, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 512, + "moe_router_topk": 10, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, + "proxy": { + "num_layers": 4, + "hidden_size": 4096, + "ffn_hidden_size": 10240, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 16, + "moe_router_topk": 2, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, +} + + +def get_qwen35_vl_language_config( + variant: str = "proxy", + **overrides, +) -> TransformerConfig: + """TransformerConfig for the Qwen3.5-VL language decoder. + + The ``397b_a17b`` variant reproduces the MIMO + ``get_qwen35_language_model_config()`` output exactly. + + Args: + variant: One of ``0.8b``, ``2b``, ``4b``, ``9b``, ``27b``, + ``35b_a3b``, ``122b_a10b``, ``397b_a17b``, + ``35b_a3b_light``, ``proxy``. + **overrides: Override any TransformerConfig field. + + Returns: + Fully-populated TransformerConfig. + """ + if variant not in _VARIANT_CONFIGS: + raise ValueError( + f"Unknown variant '{variant}'. " + f"Choose from {list(_VARIANT_CONFIGS.keys())}" + ) + + v = _VARIANT_CONFIGS[variant] + + kwargs = dict( + # Architecture + num_layers=v["num_layers"], + hidden_size=v["hidden_size"], + ffn_hidden_size=v["ffn_hidden_size"], + num_attention_heads=v["num_attention_heads"], + num_query_groups=v["num_query_groups"], + kv_channels=v["kv_channels"], + # Normalization & activation + normalization="RMSNorm", + layernorm_epsilon=1e-6, + layernorm_zero_centered_gamma=True, + apply_residual_connection_post_layernorm=False, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + # MRoPE section (interleaved T/H/W layout, Qwen3.5-VL style) + mrope_section=list(MROPE_SECTION), + mrope_interleaved=True, + rotary_interleaved=False, + # Attention + qk_layernorm=True, + attention_output_gate=True, + attention_dropout=0.0, + hidden_dropout=0.0, + add_bias_linear=False, + # Hybrid attention (GatedDeltaNet) + experimental_attention_variant="gated_delta_net", + linear_attention_freq=4, + linear_conv_kernel_dim=4, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=v["linear_num_value_heads"], + # Kernel / TE fusions + bias_activation_fusion=True, + masked_softmax_fusion=True, + persist_layer_norm=True, + bias_dropout_fusion=True, + apply_rope_fusion=False, + # Precision + bf16=True, + ) + + # MoE config (only for MoE variants) + if v["num_moe_experts"] is not None: + kwargs.update( + num_moe_experts=v["num_moe_experts"], + moe_router_topk=v["moe_router_topk"], + moe_ffn_hidden_size=v["moe_ffn_hidden_size"], + moe_shared_expert_intermediate_size=v[ + "moe_shared_expert_intermediate_size" + ], + moe_shared_expert_gate=True, + moe_layer_freq=1, + moe_router_pre_softmax=False, + moe_router_load_balancing_type="global_aux_loss", + moe_permute_fusion=True, + moe_aux_loss_coeff=1e-3, + moe_grouped_gemm=True, + moe_token_dispatcher_type="alltoall", + moe_router_dtype="fp32", + ) + + kwargs.update(overrides) + return TransformerConfig(**kwargs) diff --git a/examples/multimodal_dev/models/qwen35_vl/factory.py b/examples/multimodal_dev/models/qwen35_vl/factory.py new file mode 100644 index 00000000000..3064bc5b7f4 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/factory.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Factory functions for Qwen3.5-VL model construction. + +Encapsulates all Qwen3.5-VL-specific logic needed by ``pretrain_multimodal.py`` +so that the training entry point remains model-agnostic. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + MROPE_SECTION, + VISION_KWARGS, +) + + +def post_language_config(language_config, args): + """Apply Qwen3.5-VL-specific settings to the language TransformerConfig. + + Called after ``core_transformer_config_from_args`` to inject model-specific + fields that cannot be expressed via CLI args alone. + """ + language_config.mrope_section = list(MROPE_SECTION) + language_config.mrope_interleaved = True + + +def set_vision_flops_metadata(args, language_config, vision_config): + """Expose Qwen3.5-VL vision-model dimensions for FLOPs estimation.""" + args.count_vision_model_flops = True + args.vision_flops_variant = "qwen35_vl_v2" + args.vision_num_layers = vision_config.num_layers + args.vision_hidden_size = vision_config.hidden_size + args.vision_ffn_hidden_size = vision_config.ffn_hidden_size + args.vision_num_attention_heads = vision_config.num_attention_heads + args.vision_kv_channels = vision_config.kv_channels + args.vision_in_channels = VISION_KWARGS["in_channels"] + args.vision_patch_size = VISION_KWARGS["patch_size"] + args.vision_temporal_patch_size = VISION_KWARGS["temporal_patch_size"] + args.vision_spatial_merge_size = VISION_KWARGS["spatial_merge_size"] + args.vision_out_hidden_size = language_config.hidden_size + + +def build_model(args, language_config, vision_config, **kwargs): + """Build a complete Qwen3.5-VL model instance. + + Handles language spec construction, optional MTP block spec, and + model instantiation with Qwen3.5-VL-specific parameters. + + Args: + args: Megatron parsed arguments. + language_config: ``TransformerConfig`` for the language decoder + (already post-processed by :func:`post_language_config`). + vision_config: ``TransformerConfig`` for the vision encoder. + **kwargs: Extra keyword arguments (e.g. ``vp_stage``). + + Returns: + A :class:`Qwen35VLModel` instance. + """ + from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_mtp_block_spec, + ) + + from examples.multimodal_dev.models.qwen35_vl.model import Qwen35VLModel + from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_language_spec, + ) + + language_spec = get_qwen35_vl_language_spec( + config=language_config, + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + + mtp_block_spec = None + if getattr(args, "mtp_num_layers", None): + mtp_block_spec = get_gpt_mtp_block_spec( + config=language_config, + spec=language_spec, + use_transformer_engine=( + args.transformer_impl == "transformer_engine" + ), + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + + # When --untie-embeddings-and-output-weights is NOT passed, Megatron + # defaults to tied embeddings (share_embeddings_and_output_weights=True). + # The 0.8B variant uses tied embeddings, while larger variants untie them. + share_embeddings = not getattr( + args, "untie_embeddings_and_output_weights", False + ) + + return Qwen35VLModel( + language_config=language_config, + language_spec=language_spec, + vision_config=vision_config, + vocab_size=args.padded_vocab_size, + max_sequence_length=args.max_position_embeddings, + image_token_id=getattr(args, "image_token_id", 248056), + mtp_block_spec=mtp_block_spec, + parallel_output=True, + share_embeddings_and_output_weights=share_embeddings, + ) diff --git a/examples/multimodal_dev/models/qwen35_vl/model.py b/examples/multimodal_dev/models/qwen35_vl/model.py new file mode 100644 index 00000000000..a8fdaf67d33 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/model.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Qwen3.5-VL multimodal model for standalone FSDP + EP training. + +Composes a Megatron-native Qwen3.5 vision encoder with a ``GPTModel`` +language decoder using MRoPE and hybrid GatedDeltaNet / full-attention +layers. +""" + +from typing import Optional + +from torch import Tensor + +from examples.multimodal_dev.models.base import MultimodalModel +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, + QWEN35_VL_VOCAB_SIZE, + ROTARY_BASE, + ROTARY_PERCENT, + VISION_KWARGS, +) +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index +from examples.multimodal_dev.models.qwen35_vl.specs import get_qwen35_vl_vision_spec +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import Qwen35VLVisionEncoder +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + + +class Qwen35VLModel(MultimodalModel): + """Qwen3.5-VL multimodal model. + + Args: + language_config: ``TransformerConfig`` for the language decoder. + language_spec: ``ModuleSpec`` for language decoder layers. + vision_config: ``TransformerConfig`` for the vision encoder. + vision_spec: ``ModuleSpec`` for vision encoder layers. + vocab_size: Vocabulary size. + max_sequence_length: Maximum sequence length. + image_token_id: Token ID for image placeholders. + spatial_merge_size: Vision encoder spatial merge factor. + mtp_block_spec: Optional MTP block spec. + parallel_output: Keep outputs split across TP. + share_embeddings_and_output_weights: Tie embeddings. + """ + + def __init__( + self, + language_config: TransformerConfig, + language_spec: ModuleSpec, + vision_config: TransformerConfig, + vision_spec: ModuleSpec = None, + vocab_size: int = QWEN35_VL_VOCAB_SIZE, + max_sequence_length: int = 262144, + image_token_id: int = QWEN35_VL_IMAGE_TOKEN_ID, + video_token_id: int = QWEN35_VL_VIDEO_TOKEN_ID, + vision_start_token_id: int = QWEN35_VL_VISION_START_TOKEN_ID, + spatial_merge_size: int = 2, + mtp_block_spec: ModuleSpec = None, + parallel_output: bool = True, + share_embeddings_and_output_weights: bool = False, + ): + if vision_spec is None: + vision_spec = get_qwen35_vl_vision_spec() + + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.spatial_merge_size = spatial_merge_size + + vkw = dict(VISION_KWARGS) + vkw["spatial_merge_size"] = spatial_merge_size + vkw["out_hidden_size"] = language_config.hidden_size + + vision_encoder = Qwen35VLVisionEncoder( + config=vision_config, + transformer_layer_spec=vision_spec, + in_channels=vkw["in_channels"], + patch_size=vkw["patch_size"], + temporal_patch_size=vkw["temporal_patch_size"], + spatial_merge_size=vkw["spatial_merge_size"], + out_hidden_size=vkw["out_hidden_size"], + max_num_positions=vkw["max_num_positions"], + ) + + super().__init__( + language_config=language_config, + language_spec=language_spec, + vision_encoder=vision_encoder, + vocab_size=vocab_size, + max_sequence_length=max_sequence_length, + image_token_id=image_token_id, + position_embedding_type="mrope", + rotary_percent=ROTARY_PERCENT, + rotary_base=ROTARY_BASE, + mrope_section=language_config.mrope_section, + mtp_block_spec=mtp_block_spec, + parallel_output=parallel_output, + share_embeddings_and_output_weights=( + share_embeddings_and_output_weights + ), + ) + + def compute_position_ids( + self, + input_ids: Tensor, + image_grid_thw: Optional[Tensor] = None, + packed_seq_params=None, + ) -> Tensor: + """Compute 3D MRoPE position IDs for Qwen3.5-VL. + + In THD mode ``input_ids`` is ``[1, T]`` and ``packed_seq_params`` + supplies per-segment boundaries; positions restart at 0 per + segment. In BSHD mode ``input_ids`` is ``[B, S]`` and + ``packed_seq_params`` should be ``None``. + + Returns: + ``[3, B, S]`` position IDs for MRoPE (``[3, 1, T]`` in THD). + """ + position_ids, _ = get_rope_index( + spatial_merge_size=self.spatial_merge_size, + image_token_id=self.image_token_id, + video_token_id=self.video_token_id, + vision_start_token_id=self.vision_start_token_id, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + packed_seq_params=packed_seq_params, + ) + return position_ids diff --git a/examples/multimodal_dev/models/qwen35_vl/mrope.py b/examples/multimodal_dev/models/qwen35_vl/mrope.py new file mode 100644 index 00000000000..9e0e98b1a35 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/mrope.py @@ -0,0 +1,379 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""MRoPE (Multimodal Rotary Position Embedding) position ID computation. + +Computes 3D position IDs for Qwen3.5-VL: for text tokens all three +dimensions share sequential positions; for image/video tokens the three +dimensions encode (temporal, height, width) in the merged spatial grid. + +Supports two input layouts: + +* **BSHD** — ``input_ids`` is ``[B, S]``; each row is an independent + sample (possibly padded) and ``attention_mask`` marks valid tokens. +* **THD** — ``input_ids`` is ``[1, T]``, a concatenation of ``N`` + sub-sequences. ``packed_seq_params.cu_seqlens_q_padded`` gives the + physical segment boundaries in the packed tensor and + ``cu_seqlens_q`` gives the valid (unpadded) token count inside each + segment. Position IDs restart at 0 at every segment boundary; image + / video grid rows are consumed in packed order across segments. + +Ported from Megatron-Bridge ``get_rope_index`` (which itself is adapted +from HF ``Qwen3VLForConditionalGeneration.get_rope_index``). The inner +loop iterates over vision occurrences, not individual tokens. +""" + +from typing import Optional + +import torch +from torch import Tensor + +from megatron.core.packed_seq_params import PackedSeqParams + + +def _build_sample_mrope_positions( + sample_input_ids: Tensor, + image_grid_thw: Optional[Tensor], + video_grid_thw: Optional[Tensor], + image_index: int, + video_index: int, + spatial_merge_size: int, + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, +) -> tuple[Tensor, int, int]: + """Compute MRoPE position IDs for a single sub-sequence. + + Walks vision occurrences in ``sample_input_ids`` and produces a + ``[3, L]`` position tensor whose values start at 0. Advances + ``image_index`` / ``video_index`` through ``image_grid_thw`` / + ``video_grid_thw`` so callers can keep a running cursor across + multiple sub-sequences. + """ + vision_start_indices = torch.argwhere( + sample_input_ids == vision_start_token_id, + ).squeeze(1) + vision_tokens = sample_input_ids[vision_start_indices + 1] + image_nums = int((vision_tokens == image_token_id).sum()) + video_nums = int((vision_tokens == video_token_id).sum()) + # TODO: fuse into a kernel to drop the per-iter GPU<->CPU sync. + input_tokens = sample_input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + + text_len + + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + if llm_pos_ids_list: + positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + else: + positions = torch.zeros( + 3, 0, + dtype=sample_input_ids.dtype, + device=sample_input_ids.device, + ) + return positions, image_index, video_index + + +def get_rope_index( + spatial_merge_size: int, + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, + input_ids: Optional[Tensor] = None, + image_grid_thw: Optional[Tensor] = None, + video_grid_thw: Optional[Tensor] = None, + attention_mask: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, +) -> tuple[Tensor, Tensor]: + """Compute 3D MRoPE position IDs for Qwen3-VL / Qwen3.5-VL. + + Qwen3-VL uses timestamps rather than absolute time position IDs. + + For text tokens all three dimensions share sequential positions. + For vision tokens the three dimensions encode (temporal, height, + width) in the merged spatial grid. + + Args: + spatial_merge_size: Merge factor for spatial dimensions. + image_token_id: Token ID for image placeholders. + video_token_id: Token ID for video placeholders. + vision_start_token_id: Token ID marking start of a vision region. + input_ids: ``[B, S]`` in BSHD or ``[1, T]`` in THD. + image_grid_thw: ``[num_images, 3]`` per-image + ``(temporal, height, width)`` in patch-grid units. Rows are + consumed in the order their image tokens appear in + ``input_ids`` (packed order across segments in THD). + video_grid_thw: ``[num_videos, 3]`` per-video grid dimensions. + attention_mask: ``[B, S]`` mask (1 = keep, 0 = pad). BSHD only. + packed_seq_params: When provided, selects the THD branch and + supplies segment boundaries via ``cu_seqlens_q`` (valid + lengths) and ``cu_seqlens_q_padded`` (packed layout). + + Returns: + ``(position_ids, mrope_position_deltas)`` where *position_ids* + has shape ``[3, B, S]`` (``[3, 1, T]`` in THD). + """ + if video_grid_thw is not None: + video_grid_thw = torch.repeat_interleave( + video_grid_thw, video_grid_thw[:, 0], dim=0, + ) + video_grid_thw[:, 0] = 1 + + # ----------------------------------------------------------------- + # THD (packed) branch + # ----------------------------------------------------------------- + if packed_seq_params is not None and input_ids is not None: + cu_seqlens = packed_seq_params.cu_seqlens_q + cu_seqlens_padded = getattr( + packed_seq_params, "cu_seqlens_q_padded", None, + ) + if cu_seqlens_padded is None: + cu_seqlens_padded = cu_seqlens + + assert ( + input_ids.dim() == 2 and input_ids.shape[0] == 1 + ), "THD get_rope_index expects input_ids shape [1, T]" + + total_tokens = input_ids.shape[1] + device = input_ids.device + + # Padding slots default to 1 (matches BSHD convention where + # masked positions get filled with 1). + position_ids = torch.ones( + 3, 1, total_tokens, + dtype=input_ids.dtype, device=device, + ) + deltas: list = [] + image_index = 0 + video_index = 0 + num_segs = cu_seqlens.numel() - 1 + + for k in range(num_segs): + seg_start = int(cu_seqlens_padded[k].item()) + valid_len = int( + cu_seqlens[k + 1].item() - cu_seqlens[k].item() + ) + valid_end = seg_start + valid_len + + if valid_len == 0: + deltas.append(0) + continue + + sample_input_ids = input_ids[0, seg_start:valid_end] + + if ( + image_grid_thw is not None + or video_grid_thw is not None + ): + ( + positions, + image_index, + video_index, + ) = _build_sample_mrope_positions( + sample_input_ids=sample_input_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_index=image_index, + video_index=video_index, + spatial_merge_size=spatial_merge_size, + image_token_id=image_token_id, + video_token_id=video_token_id, + vision_start_token_id=vision_start_token_id, + ) + else: + positions = ( + torch.arange(valid_len, device=device) + .view(1, -1) + .expand(3, -1) + ) + + position_ids[:, 0, seg_start:valid_end] = positions.to( + device=device, dtype=position_ids.dtype, + ) + + if positions.numel() > 0: + deltas.append( + int(positions.max().item()) + 1 - valid_len + ) + else: + deltas.append(0) + + mrope_position_deltas = torch.tensor( + deltas, device=device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # ----------------------------------------------------------------- + # BSHD branch with vision + # ----------------------------------------------------------------- + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + elif attention_mask.dim() > 2: + attention_mask = attention_mask.any(dim=-1) + if attention_mask.dim() == 3: + attention_mask = attention_mask.squeeze(1) + attention_mask = attention_mask.to(dtype=total_input_ids.dtype) + + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + mrope_position_deltas = [] + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + + for i, sample_input_ids in enumerate(total_input_ids): + sample_input_ids = sample_input_ids[attention_mask[i] == 1] + ( + llm_positions, + image_index, + video_index, + ) = _build_sample_mrope_positions( + sample_input_ids=sample_input_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_index=image_index, + video_index=video_index, + spatial_merge_size=spatial_merge_size, + image_token_id=image_token_id, + video_token_id=video_token_id, + vision_start_token_id=vision_start_token_id, + ) + position_ids[ + ..., i, attention_mask[i] == 1 + ] = llm_positions.to(position_ids.device) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]), + ) + + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=total_input_ids.device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # ----------------------------------------------------------------- + # Text-only fallback + # ----------------------------------------------------------------- + if attention_mask is not None: + if attention_mask.dim() > 2: + attention_mask = attention_mask.any(dim=-1) + if attention_mask.dim() == 3: + attention_mask = attention_mask.squeeze(1) + attention_mask = attention_mask.to(dtype=torch.long) + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = ( + position_ids.max(0, keepdim=False)[0] + .max(-1, keepdim=True)[0] + ) + mrope_position_deltas = ( + max_position_ids + 1 - attention_mask.shape[-1] + ) + else: + position_ids = ( + torch.arange( + input_ids.shape[1], device=input_ids.device, + ) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas diff --git a/examples/multimodal_dev/models/qwen35_vl/specs.py b/examples/multimodal_dev/models/qwen35_vl/specs.py new file mode 100644 index 00000000000..22fb4e616eb --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/specs.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Layer spec helpers for Qwen3.5-VL vision encoder and language decoder. + +Provides ModuleSpec builders that define the transformer layer composition. +Both the standalone and MIMO training paths import from here. +""" + +from typing import Optional + +from examples.multimodal_dev.models.base import _NO_CP_GROUP +from megatron.core.models.gpt.experimental_attention_variant_module_specs import ( + get_transformer_block_with_experimental_attention_variant_spec, +) +from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec +from megatron.core.transformer.attention import SelfAttention +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_block import TransformerBlockSubmodules +from megatron.core.transformer.transformer_config import TransformerConfig + + +def _apply_rope_fp32(t, freqs, config, cu_seqlens=None, mscale=1.0, cp_group=None): + """Apply rotary positional embedding in fp32, then cast back to original dtype. + + Mirrors ``Qwen3VLSelfAttention.apply_rotary_pos_emb_absolute`` in Megatron-Bridge + with ``apply_rotary_pos_emb_in_fp32=True``. + """ + from megatron.core import parallel_state + from megatron.core.models.common.embeddings.rope_utils import ( + _apply_rotary_pos_emb_bshd, + _apply_rotary_pos_emb_thd, + ) + + orig_dtype = t.dtype + t_fp32 = t.float() + + if cu_seqlens is None: + out = _apply_rotary_pos_emb_bshd( + t_fp32, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=getattr(config, 'multi_latent_attention', False), + mscale=mscale, + ) + else: + if cp_group is None: + cp_group = parallel_state.get_context_parallel_group() + out = _apply_rotary_pos_emb_thd( + t_fp32, + cu_seqlens, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=getattr(config, 'multi_latent_attention', False), + mscale=mscale, + cp_group=cp_group, + ) + return out.to(orig_dtype) + + +def _apply_rope_fp32_no_cp(t, freqs, config, cu_seqlens=None, mscale=1.0, cp_group=None): + """Same as ``_apply_rope_fp32`` but forces CP-size=1. + + The vision encoder uses THD packed sequences for variable-resolution + images. When the language model uses CP>1, the global CP group would + incorrectly split the vision seqlens. This wrapper substitutes a + trivial group so the vision RoPE sees the full packed sequence. + """ + return _apply_rope_fp32( + t, freqs, config, cu_seqlens, mscale, cp_group=_NO_CP_GROUP, + ) + + +class Qwen35VLVisionSelfAttention(SelfAttention): + """ViT self-attention with RoPE applied in fp32. + + Matches Bridge's ``Qwen3VLSelfAttention`` behaviour when + ``apply_rotary_pos_emb_in_fp32=True``: query and key are cast to float32 + before the rotary multiply and cast back to bf16 afterwards. The + monkey-patch approach avoids duplicating the 300-line ``SelfAttention.forward`` + while keeping the change local to this class. + """ + + def forward(self, *args, **kwargs): + import megatron.core.transformer.attention as _attn_mod + + _orig = _attn_mod.apply_rotary_pos_emb + _attn_mod.apply_rotary_pos_emb = _apply_rope_fp32_no_cp + try: + return super().forward(*args, **kwargs) + finally: + _attn_mod.apply_rotary_pos_emb = _orig + + +def get_qwen35_vl_language_spec( + config: TransformerConfig, + vp_stage: Optional[int] = None, + pp_rank: Optional[int] = None, +) -> TransformerBlockSubmodules: + """Transformer block spec for the Qwen3.5-VL language decoder. + + Uses the experimental attention variant infrastructure to build hybrid + GatedDeltaNet + full-attention layers with optional MoE interleaving. + + Args: + config: Language decoder TransformerConfig. + vp_stage: Virtual pipeline stage. + pp_rank: Pipeline parallel rank. + + Returns: + TransformerBlockSubmodules with per-layer specs. + """ + return get_transformer_block_with_experimental_attention_variant_spec( + config=config, + vp_stage=vp_stage, + pp_rank=pp_rank, + ) + + +def get_qwen35_vl_vision_spec() -> ModuleSpec: + """ModuleSpec for vision encoder transformer layers. + + Uses ``TEDotProductAttention`` which supports packed-sequence (THD) + attention via ``PackedSeqParams`` for variable-length images. + + ``Qwen35VLVisionSelfAttention`` replaces the default ``SelfAttention`` so + that RoPE is applied in fp32, matching Bridge's + ``apply_rotary_pos_emb_in_fp32=True`` behaviour. + """ + spec = get_vit_layer_with_transformer_engine_spec() + spec.submodules.self_attention.module = Qwen35VLVisionSelfAttention + return spec diff --git a/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py b/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py new file mode 100644 index 00000000000..d57d114374d --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py @@ -0,0 +1,593 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Megatron-native Qwen3.5-VL vision encoder. + +Architecture (matches HF ``Qwen3VLVisionModel`` exactly): + + PatchEmbed (Conv3d) + → learned position embedding (bilinear interpolation) + → 2D Vision RoPE + → TransformerBlock × N (with PackedSeqParams / THD attention) + → PatchMerger (per-token LN → spatial merge → MLP) + +Key design choices: + * ``Conv3d`` patch embedding is replicated across TP ranks (no MCore + equivalent for 3D convolutions). + * ``PatchMerger`` MLP uses ``ColumnParallelLinear`` / ``RowParallelLinear`` + for TP sharding. + * Inherits from ``VisionModule``. + * Expects pixel values in block-merge order (as produced by the HF + processor) so the merger's simple reshape is correct. +""" + +from typing import List, Optional + +import torch +import torch.nn.functional as F +from torch import Tensor + +from megatron.core.models.common.vision_module.vision_module import ( + VisionModule, +) +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.tensor_parallel.layers import ( + ColumnParallelLinear, + RowParallelLinear, +) +from megatron.core.extensions.transformer_engine import TENorm +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig + +# ------------------------------------------------------------------- +# PatchEmbed — Conv3d (replicated, no TP sharding) +# ------------------------------------------------------------------- + +class Qwen35VLPatchEmbed(MegatronModule): + """3D convolution patch embedding matching HF ``Qwen3VLVisionPatchEmbed``. + + Uses ``nn.Conv3d`` with kernel = stride = ``[temporal_patch_size, + patch_size, patch_size]`` and ``bias=True``. The module is replicated + across TP ranks (no MCore equivalent for 3D conv). + + Args: + config: TransformerConfig (used by MegatronModule base). + in_channels: Number of input channels (3 for RGB). + hidden_size: Output embedding dimension. + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + """ + + def __init__( + self, + config: TransformerConfig, + in_channels: int = 3, + hidden_size: int = 1152, + patch_size: int = 16, + temporal_patch_size: int = 2, + ): + super().__init__(config=config) + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.in_channels = in_channels + self.hidden_size = hidden_size + + kernel = [temporal_patch_size, patch_size, patch_size] + self.proj = torch.nn.Conv3d( + in_channels, + hidden_size, + kernel_size=kernel, + stride=kernel, + bias=True, + ) + + def forward(self, pixel_values: Tensor) -> Tensor: + """Forward pass. + + Args: + pixel_values: ``[total_patches, C * T * pH * pW]`` + pre-extracted flat patches. + + Returns: + Patch embeddings ``[total_patches, hidden_size]``. + """ + target_dtype = self.proj.weight.dtype + pixel_values = pixel_values.view( + -1, + self.in_channels, + self.temporal_patch_size, + self.patch_size, + self.patch_size, + ) + return self.proj(pixel_values.to(dtype=target_dtype)).view( + -1, self.hidden_size + ) + + +# ------------------------------------------------------------------- +# VisionRotaryEmbedding — 1D frequency table +# ------------------------------------------------------------------- + +class Qwen35VLVisionRotaryEmbedding(MegatronModule): + """1D rotary position frequency table for the vision transformer. + + Generates RoPE frequencies for integer positions ``0 .. seqlen-1``. + The encoder maps 2D (row, col) positions to embeddings via table + lookup. Matches HF ``Qwen3VLVisionRotaryEmbedding``. + + Args: + dim: Frequency dimension (``head_dim // 2``). + theta: RoPE base frequency. + config: Optional TransformerConfig for MegatronModule base. + """ + + def __init__( + self, + dim: int, + theta: float = 10000.0, + config: Optional[TransformerConfig] = None, + ): + super().__init__(config=config) + self.dim = dim + self.theta = theta + inv_freq = 1.0 / ( + theta + ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim) + ) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def _get_inv_freq(self, device: torch.device) -> Tensor: + """Return ``inv_freq`` in float32 on *device*. + + Always recomputes in float32 regardless of the buffer's stored dtype. + This matches Bridge's lazy-init behaviour where ``inv_freq`` is + constructed fresh (in float32) on the first forward call, after any + ``model.bfloat16()`` cast has already occurred. + """ + return 1.0 / ( + self.theta + ** ( + torch.arange( + 0, self.dim, 2, + dtype=torch.float32, device=device, + ) + / self.dim + ) + ) + + def forward( + self, + seqlen: int, + device: Optional[torch.device] = None, + ) -> Tensor: + """Frequency lookup table for positions ``0 .. seqlen-1``. + + Args: + seqlen: Number of positions. + device: Runtime device (required for meta-init safety). + + Returns: + ``[seqlen, dim // 2]`` frequencies. + """ + if device is None: + if self.inv_freq.device.type != "meta": + device = self.inv_freq.device + else: + device = torch.device( + "cuda", torch.cuda.current_device() + ) + inv_freq = self._get_inv_freq(device) + seq = torch.arange(seqlen, device=device, dtype=inv_freq.dtype) + return torch.outer(seq, inv_freq) + + +# ------------------------------------------------------------------- +# PatchMerger — per-token LN, spatial merge, TP-sharded MLP +# ------------------------------------------------------------------- + +class Qwen35VLPatchMerger(MegatronModule): + """Spatial patch merger matching HF ``Qwen3VLVisionPatchMerger``. + + Per-token ``LayerNorm`` on ``hidden_size`` → reshape to merge + ``spatial_merge_size ** 2`` adjacent patches → two-layer MLP + (``ColumnParallelLinear`` → GELU → ``RowParallelLinear``). + + MLP dimensions: ``merge_dim → merge_dim → out_hidden_size`` + where ``merge_dim = hidden_size * spatial_merge_size ** 2``. + + Args: + config: TransformerConfig (provides TP settings, init_method). + hidden_size: Per-token hidden size from the ViT. + out_hidden_size: Output dimension (language model hidden_size). + spatial_merge_size: Merge factor per spatial dimension. + """ + + def __init__( + self, + config: TransformerConfig, + hidden_size: int = 1152, + out_hidden_size: int = 3584, + spatial_merge_size: int = 2, + ): + super().__init__(config=config) + self.spatial_merge_size = spatial_merge_size + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + merge_dim = self.merge_dim + + self.patch_norm = TENorm(config=config, hidden_size=hidden_size, eps=1e-6) + self.linear_fc1 = build_module( + ColumnParallelLinear, + merge_dim, + merge_dim, + config=config, + init_method=config.init_method, + bias=True, + gather_output=False, + ) + self.linear_fc2 = build_module( + RowParallelLinear, + merge_dim, + out_hidden_size, + config=config, + init_method=config.output_layer_init_method, + bias=True, + input_is_parallel=True, + skip_bias_add=False, + ) + + def forward(self, hidden_states: Tensor) -> Tensor: + """Merge patches spatially. + + Args: + hidden_states: ``[total_patches, hidden_size]`` in block-merge + order from the ViT transformer blocks. + + Returns: + ``[total_merged_patches, out_hidden_size]``. + """ + hidden_states = self.patch_norm(hidden_states) + merged = hidden_states.view(-1, self.merge_dim) + merged, _ = self.linear_fc1(merged) + # Match official HuggingFace Qwen3VLVisionPatchMerger (default approximate='none'). + merged = torch.nn.functional.gelu(merged, approximate="none") + merged, _ = self.linear_fc2(merged) + return merged + + +# ------------------------------------------------------------------- +# Qwen35VLVisionEncoder — top-level encoder module +# ------------------------------------------------------------------- + +class Qwen35VLVisionEncoder(VisionModule): + """Megatron-native Qwen3.5-VL vision encoder. + + Processes image / video inputs through: + + 1. ``Qwen35VLPatchEmbed`` (Conv3d) + 2. Learned ``nn.Embedding`` position table with bilinear interpolation + 3. 2D Vision RoPE from ``(row, col)`` patch positions + 4. ``TransformerBlock`` × N with ``PackedSeqParams`` (THD attention) + 5. ``Qwen35VLPatchMerger`` + + Output dimension matches the language model ``hidden_size``. + + Args: + config: Vision ``TransformerConfig``. + transformer_layer_spec: ``ModuleSpec`` for ViT layers. + in_channels: Image channels (3 for RGB). + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + spatial_merge_size: Spatial merge factor. + out_hidden_size: Output dim (language decoder hidden_size). + max_num_positions: Size of the learned position table. + """ + + def __init__( + self, + config: TransformerConfig, + transformer_layer_spec: ModuleSpec = None, + in_channels: int = 3, + patch_size: int = 16, + temporal_patch_size: int = 2, + spatial_merge_size: int = 2, + out_hidden_size: int = 3584, + max_num_positions: int = 2304, + ): + super().__init__(config=config) + + self.hidden_size = config.hidden_size + self.spatial_merge_size = spatial_merge_size + + # --- Patch embedding (Conv3d) --- + self.patch_embed = Qwen35VLPatchEmbed( + config=config, + in_channels=in_channels, + hidden_size=config.hidden_size, + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + ) + + # --- Learned position embedding with bilinear interpolation --- + self.pos_embed = torch.nn.Embedding( + max_num_positions, config.hidden_size, + ) + self.num_grid_per_side = int(max_num_positions ** 0.5) + + # --- Vision rotary embeddings --- + head_dim = config.hidden_size // config.num_attention_heads + self.rot_pos_emb = Qwen35VLVisionRotaryEmbedding( + head_dim // 2, config=config, + ) + + # --- Transformer blocks --- + if transformer_layer_spec is None: + from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_vision_spec, + ) + transformer_layer_spec = get_qwen35_vl_vision_spec() + + self.decoder = TransformerBlock( + config=config, + spec=transformer_layer_spec, + pre_process=True, + post_process=True, + post_layer_norm=False, + ) + + # --- Patch merger --- + self.merger = Qwen35VLPatchMerger( + config=config, + hidden_size=config.hidden_size, + out_hidden_size=out_hidden_size, + spatial_merge_size=spatial_merge_size, + ) + + # --------------------------------------------------------------- + # Learned position embedding with bilinear interpolation + # --------------------------------------------------------------- + + def _fast_pos_embed_interpolate( + self, grid_thw: Tensor, + ) -> Tensor: + """Bilinear interpolation of the learned 2D position table. + + Matches HF ``Qwen3VLVisionModel.fast_pos_embed_interpolate``. + + Args: + grid_thw: ``[num_images, 3]`` (T, H, W) in patch-grid units. + + Returns: + ``[total_patches, hidden_size]`` position embeddings in + block-merge order. + """ + grid_thw_list = grid_thw.tolist() + grid_ts = [int(row[0]) for row in grid_thw_list] + grid_hs = [int(row[1]) for row in grid_thw_list] + grid_ws = [int(row[2]) for row in grid_thw_list] + device = self.pos_embed.weight.device + n = self.num_grid_per_side + + idx_list: List[List[int]] = [[] for _ in range(4)] + weight_list: List[List[float]] = [[] for _ in range(4)] + + for t, h, w in grid_thw_list: + t, h, w = int(t), int(h), int(w) + h_idxs = torch.linspace(0, n - 1, h) + w_idxs = torch.linspace(0, n - 1, w) + + h_floor = h_idxs.int() + w_floor = w_idxs.int() + h_ceil = (h_floor + 1).clip(max=n - 1) + w_ceil = (w_floor + 1).clip(max=n - 1) + + dh = h_idxs - h_floor.float() + dw = w_idxs - w_floor.float() + + base_h = h_floor * n + base_h_ceil = h_ceil * n + + indices = [ + (base_h[None].T + w_floor[None]).flatten(), + (base_h[None].T + w_ceil[None]).flatten(), + (base_h_ceil[None].T + w_floor[None]).flatten(), + (base_h_ceil[None].T + w_ceil[None]).flatten(), + ] + weights = [ + ((1 - dh)[None].T * (1 - dw)[None]).flatten(), + ((1 - dh)[None].T * dw[None]).flatten(), + (dh[None].T * (1 - dw)[None]).flatten(), + (dh[None].T * dw[None]).flatten(), + ] + + for i in range(4): + idx_list[i].extend(indices[i].tolist()) + weight_list[i].extend(weights[i].tolist()) + + idx_tensor = torch.tensor( + idx_list, dtype=torch.long, device=device, + ) + weight_tensor = torch.tensor( + weight_list, + dtype=self.pos_embed.weight.dtype, + device=device, + ) + pos_embeds = ( + self.pos_embed(idx_tensor).to(device) + * weight_tensor[:, :, None] + ) + patch_pos_embeds = ( + pos_embeds[0] + pos_embeds[1] + + pos_embeds[2] + pos_embeds[3] + ) + + patch_pos_embeds = patch_pos_embeds.split( + [h * w for h, w in zip(grid_hs, grid_ws)] + ) + + merge = self.spatial_merge_size + result = [] + for pe, t, h, w in zip( + patch_pos_embeds, grid_ts, grid_hs, grid_ws, + ): + pe = pe.repeat(t, 1) + pe = ( + pe.view( + t, h // merge, merge, w // merge, merge, -1, + ) + .permute(0, 1, 3, 2, 4, 5) + .flatten(0, 4) + ) + result.append(pe) + + return torch.cat(result) + + # --------------------------------------------------------------- + # 2D Vision RoPE + # --------------------------------------------------------------- + + def _compute_rotary_pos_emb(self, grid_thw: Tensor) -> Tensor: + """Compute 2D Vision RoPE for all patches in block-merge order. + + Matches HF ``Qwen3VLVisionModel.rot_pos_emb``. + + Args: + grid_thw: ``[num_images, 3]`` (T, H, W) per image. + + Returns: + ``[total_patches, head_dim // 2]`` raw RoPE frequencies. + """ + merge = self.spatial_merge_size + grid_thw_list = grid_thw.tolist() + + max_hw = max(max(int(h), int(w)) for _, h, w in grid_thw_list) + freq_table = self.rot_pos_emb( + max_hw, device=grid_thw.device, + ) + device = freq_table.device + + total_tokens = sum( + int(t) * int(h) * int(w) for t, h, w in grid_thw_list + ) + pos_ids = torch.empty( + (total_tokens, 2), dtype=torch.long, device=device, + ) + + offset = 0 + for num_frames, height, width in grid_thw_list: + num_frames = int(num_frames) + height = int(height) + width = int(width) + merged_h = height // merge + merged_w = width // merge + + block_rows = torch.arange(merged_h, device=device) + block_cols = torch.arange(merged_w, device=device) + intra_row = torch.arange(merge, device=device) + intra_col = torch.arange(merge, device=device) + + row_idx = ( + block_rows[:, None, None, None] * merge + + intra_row[None, None, :, None] + ) + col_idx = ( + block_cols[None, :, None, None] * merge + + intra_col[None, None, None, :] + ) + + row_idx = row_idx.expand( + merged_h, merged_w, merge, merge, + ).reshape(-1) + col_idx = col_idx.expand( + merged_h, merged_w, merge, merge, + ).reshape(-1) + + coords = torch.stack((row_idx, col_idx), dim=-1) + if num_frames > 1: + coords = coords.repeat(num_frames, 1) + + n_tokens = coords.shape[0] + pos_ids[offset: offset + n_tokens] = coords + offset += n_tokens + + embeddings = freq_table[pos_ids] + embeddings = embeddings.flatten(1) + return embeddings + + # --------------------------------------------------------------- + # PackedSeqParams for variable-length attention + # --------------------------------------------------------------- + + @staticmethod + def _build_packed_seq_params(grid_thw: Tensor) -> PackedSeqParams: + """Build ``PackedSeqParams`` from grid dimensions. + + Each temporal frame of each image forms a separate sub-sequence + in the packed THD layout, matching HF's ``cu_seqlens`` computation. + + Args: + grid_thw: ``[num_images, 3]``. + + Returns: + ``PackedSeqParams`` for ``TransformerBlock``. + """ + cu_seqlens = torch.repeat_interleave( + grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0], + ).cumsum(dim=0, dtype=torch.int32) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + max_seqlen = int( + (grid_thw[:, 1] * grid_thw[:, 2]).max().item() + ) + + return PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + ) + + # --------------------------------------------------------------- + # Forward + # --------------------------------------------------------------- + + def forward( + self, + pixel_values: Tensor, + grid_thw: Tensor, + ) -> Tensor: + """Encode images / video frames. + + Args: + pixel_values: ``[total_patches, C * T * pH * pW]`` + pre-extracted flat patches in block-merge order. + grid_thw: ``[num_images, 3]`` (T, H, W) in patch-grid units. + + Returns: + ``[total_merged_patches, out_hidden_size]`` visual embeddings. + """ + # 1. Patch embedding (Conv3d) + hidden_states = self.patch_embed(pixel_values) + + # 2. Learned position embedding (bilinear interpolation) + pos_embeds = self._fast_pos_embed_interpolate(grid_thw) + hidden_states = hidden_states + pos_embeds + + # 3. 2D Vision RoPE + rot_freqs = self._compute_rotary_pos_emb(grid_thw) + emb = torch.cat((rot_freqs, rot_freqs), dim=-1) + rot_freqs_expanded = emb.unsqueeze(1).unsqueeze(1) + + # 4. Transformer blocks with PackedSeqParams + packed_seq_params = self._build_packed_seq_params(grid_thw) + hidden_states = hidden_states.unsqueeze(1) + hidden_states = self.decoder( + hidden_states=hidden_states, + attention_mask=None, + rotary_pos_emb=rot_freqs_expanded, + packed_seq_params=packed_seq_params, + ) + hidden_states = hidden_states.squeeze(1) + + # 5. Patch merger + return self.merger(hidden_states) diff --git a/examples/multimodal_dev/pretrain_multimodal.py b/examples/multimodal_dev/pretrain_multimodal.py new file mode 100644 index 00000000000..053fa00a5a2 --- /dev/null +++ b/examples/multimodal_dev/pretrain_multimodal.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Standalone entry point for multimodal_dev model training (FSDP + EP). + +This entry point is **model-agnostic**. All model-specific logic (layer +specs, model construction, FLOPs metadata, dataset generation) is +delegated to factory functions registered in +:data:`multimodal_dev.models.MODEL_REGISTRY`. + +Adding a new architecture only requires: + +1. Creating a new model package under ``multimodal_dev/models//`` + with the appropriate factory functions. +2. Registering an entry in ``MODEL_REGISTRY``. + +No changes to this file are necessary. + +Usage:: + + torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \\ + --model-arch qwen35_vl \\ + --dataset-provider mock \\ + ... (other megatron args) +""" + +import importlib +import os +import sys + +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")), +) + +from examples.multimodal_dev.arguments import add_multimodal_args +from examples.multimodal_dev.forward_step import forward_step +from megatron.core.enums import ModelType +from megatron.training import get_args, pretrain +from megatron.training.argument_utils import pretrain_cfg_container_from_args +from megatron.training.arguments import core_transformer_config_from_args, parse_and_validate_args + + +def model_provider( + pre_process: bool = True, + post_process: bool = True, + **kwargs, +): + """Build a multimodal model from ``--model-arch``. + + The language ``TransformerConfig`` is built from CLI args so that + parallelism settings, precision, and fusion flags are inherited. + Model-specific post-processing and construction are delegated to the + registry factory functions. + """ + args = get_args() + model_arch = getattr(args, "model_arch", "qwen35_vl") + + from examples.multimodal_dev.models import MODEL_REGISTRY + + if model_arch not in MODEL_REGISTRY: + raise ValueError( + f"Unknown model arch '{model_arch}'. " + f"Available: {list(MODEL_REGISTRY.keys())}" + ) + + registry = MODEL_REGISTRY[model_arch] + + # --- language config (generic + model-specific post-processing) --- + language_config = core_transformer_config_from_args(args) + post_language_config_fn = registry.get("post_language_config_fn") + if post_language_config_fn is not None: + post_language_config_fn(language_config, args) + + # --- vision config --- + vision_config = registry["vision_config_fn"]( + num_layers_override=getattr(args, "vision_num_layers", None), + variant=getattr(args, "model_variant", None), + ) + vision_config.bf16 = language_config.bf16 + vision_config.fp16 = language_config.fp16 + + if getattr(args, "recompute_vision", False): + vision_config.recompute_granularity = "full" + vision_config.recompute_method = "uniform" + vision_config.recompute_num_layers = 1 + + # --- vision FLOPs metadata --- + vision_flops_fn = registry.get("vision_flops_fn") + if vision_flops_fn is not None: + vision_flops_fn(args, language_config, vision_config) + + # --- build model (fully delegated to the arch factory) --- + model = registry["model_factory_fn"]( + args=args, + language_config=language_config, + vision_config=vision_config, + **kwargs, + ) + + return model + + +def _resolve_provider_fn(provider_fn): + """Resolve a provider that may be a dotted import path string.""" + if isinstance(provider_fn, str): + module_path, func_name = provider_fn.rsplit(".", 1) + provider_fn = getattr( + importlib.import_module(module_path), func_name, + ) + return provider_fn + + +def datasets_provider(train_val_test_num_samples): + """Dataset provider dispatcher. + + Routes to the dataset factory registered for the current + ``(--model-arch, --dataset-provider)`` combination. + """ + args = get_args() + model_arch = getattr(args, "model_arch", "qwen35_vl") + provider = getattr(args, "dataset_provider", "mock") + + from examples.multimodal_dev.models import MODEL_REGISTRY + + if model_arch not in MODEL_REGISTRY: + raise ValueError( + f"Unknown model arch '{model_arch}'. " + f"Available: {list(MODEL_REGISTRY.keys())}" + ) + + registry = MODEL_REGISTRY[model_arch] + available = registry.get("dataset_providers", {}) + + if provider not in available: + raise ValueError( + f"Unknown dataset provider '{provider}' for arch " + f"'{model_arch}'. Available: {list(available.keys())}" + ) + + provider_fn = _resolve_provider_fn(available[provider]) + return provider_fn(train_val_test_num_samples) + + +if __name__ == "__main__": + datasets_provider.is_distributed = True + + args = parse_and_validate_args( + extra_args_provider=add_multimodal_args, + args_defaults={}, + ) + # multimodal_dev's model_provider builds the full model on every rank and + # does not honor pre_process / post_process pipeline-stage flags. PP>1 + # would silently violate Megatron's pipeline-parallel contract. + if args.pipeline_model_parallel_size > 1: + raise ValueError( + "multimodal_dev does not support pipeline_model_parallel_size > 1 " + f"(got {args.pipeline_model_parallel_size}). The model provider " + "builds the full model on every rank; pipeline-stage splitting is " + "not wired through. Run with --pipeline-model-parallel-size 1." + ) + full_config = pretrain_cfg_container_from_args(args) + pretrain( + full_config, + datasets_provider, + model_provider, + ModelType.encoder_or_decoder, + forward_step, + ) diff --git a/examples/multimodal_dev/scripts/run_qwen35_vl.sh b/examples/multimodal_dev/scripts/run_qwen35_vl.sh new file mode 100755 index 00000000000..3a1ca55c826 --- /dev/null +++ b/examples/multimodal_dev/scripts/run_qwen35_vl.sh @@ -0,0 +1,526 @@ +#!/bin/bash + +# Launch script for Qwen3.5-VL training via multimodal_dev (FSDP + EP). +# +# Usage (from the Megatron-LM repo root): +# ./examples/multimodal_dev/scripts/run_qwen35_vl.sh +# +# Environment variables: +# MODEL_VARIANT: proxy (default), 0.8b, 2b, 4b, 9b, 27b, 35b_a3b, 122b_a10b, 397b_a17b, 35b_a3b_light +# CKPT_LOAD: path to a pre-converted checkpoint to load (enables --load + --finetune) +# CKPT_FORMAT: checkpoint format override (e.g. torch_dist); auto-detected when empty +# TP, EP, PP: parallelism sizes (PP must stay 1; multimodal_dev does not +# support pipeline parallelism) +# MBS, GBS: micro/global batch sizes +# NUM_LAYERS, NUM_EXPERTS: override for proxy testing +# FORCE_LOAD_BALANCING: set to 1 to enable --moe-router-force-load-balancing +# (perf / mock-data only; OFF for real finetuning) +# LAUNCHER: torchrun (default) or python +# PROFILE: set to 1 to enable Nsight Systems profiling (default: 0) +# PROFILE_STEP_START/PROFILE_STEP_END: profiled iteration window (default: 4-5) + +# example script: +# DRY_RUN=0 MODEL_VARIANT=proxy USE_PACKED_SEQUENCE=1 bash ./examples/multimodal_dev/scripts/run_qwen35_vl.sh + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NCCL_IB_SL=1 +export NVTE_FUSED_ATTN=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +DRY_RUN=${DRY_RUN:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +if [ -n "${SLURM_JOB_NUM_NODES:-}" ]; then + NUM_NODES="$SLURM_JOB_NUM_NODES" +else + NUM_NODES=${NNODES:-1} +fi +PROFILE=${PROFILE:-0} +PROFILE_STEP_START=${PROFILE_STEP_START:-4} +PROFILE_STEP_END=${PROFILE_STEP_END:-5} +PROFILE_RANKS=${PROFILE_RANKS:-0} +LAUNCHER=${LAUNCHER:-torchrun} + +MODEL_VARIANT=${MODEL_VARIANT:-proxy} +VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-} + +# Batch sizes +MBS=${MBS:-2} +GBS=${GBS:-16} + +# Parallelism +TP=${TP:-1} +# EP defaults to 1; MoE variants override via the variant case block below. +EP=${EP:-1} +PP=${PP:-1} +CP=${CP:-1} +# Gate --moe-router-force-load-balancing behind an explicit opt-in. Useful for +# perf / mock-data benchmarking (it disables the auxiliary load-balancing loss +# coupling so router routes are perfectly uniform), but it must be off for any +# real fine-tuning / convergence run because it freezes data-dependent routing. +FORCE_LOAD_BALANCING=${FORCE_LOAD_BALANCING:-0} + +# Variant-aware architecture defaults. +# The model provider builds configs from the variant dict in +# multimodal_dev/models/qwen35_vl/configuration.py, but Megatron also +# uses these CLI args internally (PP splits, param counting). +case "$MODEL_VARIANT" in + 0.8b) + NUM_LAYERS=${NUM_LAYERS:-24} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=1024 + FFN_HIDDEN_SIZE=3584 + NUM_ATTN_HEADS=8 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=16 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-12} + ;; + 2b) + NUM_LAYERS=${NUM_LAYERS:-24} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=6144 + NUM_ATTN_HEADS=8 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=16 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-24} + ;; + 4b) + NUM_LAYERS=${NUM_LAYERS:-32} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=2560 + FFN_HIDDEN_SIZE=9216 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-24} + ;; + proxy) + NUM_LAYERS=${NUM_LAYERS:-4} + NUM_EXPERTS=${NUM_EXPERTS:-16} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=10240 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-2} + ;; + 9b) + NUM_LAYERS=${NUM_LAYERS:-32} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=12288 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 27b) + NUM_LAYERS=${NUM_LAYERS:-64} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=5120 + FFN_HIDDEN_SIZE=17408 + NUM_ATTN_HEADS=24 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=48 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 35b_a3b) + NUM_LAYERS=${NUM_LAYERS:-40} + NUM_EXPERTS=${NUM_EXPERTS:-256} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=4096 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 35b_a3b_light) + NUM_LAYERS=${NUM_LAYERS:-12} + NUM_EXPERTS=${NUM_EXPERTS:-128} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=4096 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-7} + ;; + 122b_a10b) + NUM_LAYERS=${NUM_LAYERS:-48} + NUM_EXPERTS=${NUM_EXPERTS:-256} + HIDDEN_SIZE=3072 + FFN_HIDDEN_SIZE=8192 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 397b_a17b) + NUM_LAYERS=${NUM_LAYERS:-60} + NUM_EXPERTS=${NUM_EXPERTS:-512} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=10240 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + *) + : "${NUM_LAYERS:?NUM_LAYERS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_EXPERTS:?NUM_EXPERTS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${HIDDEN_SIZE:?HIDDEN_SIZE must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${FFN_HIDDEN_SIZE:?FFN_HIDDEN_SIZE must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_ATTN_HEADS:?NUM_ATTN_HEADS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_QUERY_GROUPS:?NUM_QUERY_GROUPS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${LINEAR_NUM_VALUE_HEADS:?LINEAR_NUM_VALUE_HEADS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; +esac + +# Fail fast on inconsistent expert-parallelism configuration. Dense variants +# (NUM_EXPERTS=0) do not emit any --num-experts / MoE args, so forwarding +# --expert-model-parallel-size > 1 would trip Megatron's arg validation. +if [ "${NUM_EXPERTS:-0}" -eq 0 ] && [ "$EP" -gt 1 ]; then + echo "ERROR: MODEL_VARIANT=$MODEL_VARIANT has NUM_EXPERTS=0 (dense) but EP=$EP." >&2 + echo " Set EP=1 for dense variants, or pick a MoE variant." >&2 + exit 1 +fi + +SEQ_LEN=${SEQ_LEN:-4096} + +WANDB_PROJECT=${WANDB_PROJECT:-'qwen35-vl-0524'} +EXP_NAME="qwen35vl_${MODEL_VARIANT}_tp${TP}_ep${EP}_pp${PP}_cp${CP}" + +RECOMPUTE_VISION=${RECOMPUTE_VISION:-0} +if [ "$RECOMPUTE_VISION" -eq 1 ]; then + EXP_NAME+="_recompute_encoder" +fi +RECOMPUTE=${RECOMPUTE:-0} +if [ "$RECOMPUTE" -eq 1 ]; then + EXP_NAME+="_recompute_decoder" +fi + +USE_PACKED_SEQUENCE=${USE_PACKED_SEQUENCE:-0} +if [ "$USE_PACKED_SEQUENCE" -eq 1 ]; then + EXP_NAME+="_thd" +fi + +MEGATRON_LM_PATH="${MEGATRON_LM_PATH:-$(cd "$(dirname "$0")/../../.." && pwd)}" +ROOT_DIR="${ROOT_DIR:-${MEGATRON_LM_PATH}/local/}" +CHECKPOINT_STORE_PATH="${ROOT_DIR}${EXP_NAME}" +mkdir -p "$CHECKPOINT_STORE_PATH" + +TENSORBOARD_LOGS_PATH="${TENSORBOARD_LOGS_PATH:-${MEGATRON_LM_PATH}/logs}" +mkdir -p "$TENSORBOARD_LOGS_PATH" + +DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --nnodes "$NUM_NODES" +) + +if [ "$NUM_NODES" -gt 1 ]; then + DISTRIBUTED_ARGS+=( + --master_addr "${MASTER_ADDR:-localhost}" + --master_port "${MASTER_PORT:-6000}" + ) +fi + +# --- Parallelism --- +MODEL_PARALLEL_ARGS=( + --tensor-model-parallel-size "$TP" + --pipeline-model-parallel-size "$PP" + --expert-model-parallel-size "$EP" + --context-parallel-size "$CP" + --cp-comm-type "a2a" + --expert-tensor-parallel-size 1 + --use-distributed-optimizer + --sequence-parallel +) + +# --- Training --- +TRAINING_ARGS=( + --micro-batch-size "$MBS" + --global-batch-size "$GBS" + --train-iters "${TRAIN_ITERS:-500}" + --adam-beta1 0.9 + --adam-beta2 0.95 + --lr 1.2e-4 + --min-lr 1.2e-5 + --lr-decay-style cosine + --lr-warmup-iters 100 + --lr-decay-iters 2000 + --weight-decay 0.1 + --clip-grad 1.0 + --bf16 + --use-mcore-models + --transformer-impl transformer_engine + --cross-entropy-loss-fusion + --cross-entropy-fusion-impl te + --enable-experimental + --manual-gc + --manual-gc-interval 50 + --mtp-num-layers 1 + --mtp-loss-scaling-factor 0.1 + --sft + --use-flash-attn + # --attention-backend flash + --calculate-per-token-loss +) + +PROFILE_ARGS=() +NSYS_CMD=() +if [ "$PROFILE" = "1" ]; then + PROFILE_ARGS=( + --profile + --profile-step-start "$PROFILE_STEP_START" + --profile-step-end "$PROFILE_STEP_END" + --profile-ranks "$PROFILE_RANKS" + ) + + NSYS_OUTPUT_DIR="${CHECKPOINT_STORE_PATH}/nsys" + mkdir -p "$NSYS_OUTPUT_DIR" + NSYS_CMD=( + nsys profile + --sample=none + --cpuctxsw=none + --trace=cuda,nvtx,cublas,cudnn + --force-overwrite=true + --capture-range=cudaProfilerApi + --capture-range-end=stop + -o "${NSYS_OUTPUT_DIR}/${EXP_NAME}_$(date +%Y%m%d_%H%M%S)" + ) +fi + +# --- Logging & Checkpointing --- +SAVE_INTERVAL=${SAVE_INTERVAL:-500} +EVAL_AND_LOGGING_ARGS=( + --log-interval 1 + --save-interval "$SAVE_INTERVAL" + --eval-interval 500 + --save "$CHECKPOINT_STORE_PATH" + --eval-iters 10 + --tensorboard-dir "$TENSORBOARD_LOGS_PATH" + --wandb-project "$WANDB_PROJECT" + --wandb-exp-name "$EXP_NAME" + --wandb-save-dir "$CHECKPOINT_STORE_PATH" + --log-throughput + --log-timers-to-tensorboard + --log-params-norm +) + +# --- Tokenizer --- +TOKENIZER_MODEL=${TOKENIZER_MODEL:-Qwen/Qwen3.5-397B-A17B} +TOKENIZER_ARGS=( + --tokenizer-type HuggingFaceTokenizer + --tokenizer-model "$TOKENIZER_MODEL" +) + +# --- Multimodal-specific --- +MULTIMODAL_ARGS=( + --model-arch qwen35_vl + --model-variant "$MODEL_VARIANT" + --dataset-provider cord_v2 + --hf-processor-path Qwen/Qwen3.5-397B-A17B + --use-vanilla-collate-fn + --image-token-id 248056 + --image-size 224 + --total-seq-length "$SEQ_LEN" + --image-seq-length 256 + --vision-num-layers "$VISION_NUM_LAYERS" +) + +if [ "$USE_PACKED_SEQUENCE" -eq 1 ]; then + MULTIMODAL_ARGS+=( --use-packed-sequence ) +fi + +# --- Qwen3.5 Decoder Architecture (variant-specific dims set above) --- +# These must match examples/multimodal_dev/models/qwen35_vl/configuration.py +GPT_MODEL_ARGS=( + --num-layers "$NUM_LAYERS" + --hidden-size "$HIDDEN_SIZE" + --ffn-hidden-size "$FFN_HIDDEN_SIZE" + --num-attention-heads "$NUM_ATTN_HEADS" + --group-query-attention + --num-query-groups "$NUM_QUERY_GROUPS" + --kv-channels 256 + --max-position-embeddings 262144 + --seq-length "$SEQ_LEN" + --normalization RMSNorm + --apply-layernorm-1p + --norm-epsilon 1e-06 + --swiglu + --disable-bias-linear + --position-embedding-type rope + --rotary-percent 0.25 + --rotary-base 10000000 + --rotary-seq-len-interpolation-factor 1 + --qk-layernorm + --attention-output-gate + --attention-dropout 0.0 + --hidden-dropout 0.0 + --experimental-attention-variant gated_delta_net + --linear-attention-freq 4 + --linear-conv-kernel-dim 4 + --linear-key-head-dim 128 + --linear-value-head-dim 128 + --linear-num-key-heads 16 + --linear-num-value-heads "$LINEAR_NUM_VALUE_HEADS" + --make-vocab-size-divisible-by 485 +) + +# --- Tied / untied embeddings --- +# 0.8B, 2B, 4B use tied embeddings; all other variants untie them. +case "$MODEL_VARIANT" in + 0.8b|2b|4b) ;; + *) GPT_MODEL_ARGS+=( --untie-embeddings-and-output-weights ) ;; +esac + +# --- MoE args (MoE variants only) --- +MOE_ARGS=() +case "$MODEL_VARIANT" in + proxy) + MOE_TOPK=2; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 35b_a3b|35b_a3b_light) + MOE_TOPK=8; MOE_FFN_HIDDEN=512; MOE_SHARED_HIDDEN=512 + ;; + 122b_a10b) + MOE_TOPK=8; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 397b_a17b) + MOE_TOPK=10; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 0.8b|2b|4b|9b|27b) + ;; +esac +if [ "${NUM_EXPERTS:-0}" -gt 0 ]; then + MOE_ARGS=( + --num-experts "$NUM_EXPERTS" + --moe-ffn-hidden-size "$MOE_FFN_HIDDEN" + --moe-shared-expert-intermediate-size "$MOE_SHARED_HIDDEN" + --moe-shared-expert-gate + --moe-router-load-balancing-type aux_loss + --moe-router-topk "$MOE_TOPK" + --moe-grouped-gemm + --moe-aux-loss-coeff 1e-3 + --moe-token-dispatcher-type alltoall + --moe-router-dtype fp32 + --moe-permute-fusion + --moe-router-fusion + ) + # Perf / mock-data only: forces uniform router decisions; do NOT enable for + # real finetuning (it freezes data-dependent routing). + if [ "$FORCE_LOAD_BALANCING" -eq 1 ]; then + MOE_ARGS+=( --moe-router-force-load-balancing ) + fi +fi + +# --- Recompute --- +if [ "$RECOMPUTE" -eq 1 ]; then + RECOMPUTE_ARGS=( + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + ) + # RECOMPUTE_ARGS=( + # --recompute-granularity selective + # --recompute-modules moe_act shared_experts layernorm moe + # ) +else + RECOMPUTE_ARGS=() +fi +if [ "$RECOMPUTE_VISION" -eq 1 ]; then + RECOMPUTE_ARGS+=( --recompute-vision ) +fi + +# --- Checkpoint loading --- +# CKPT_LOAD: path to checkpoint directory +# CKPT_FORMAT: override checkpoint format (default: auto-detect) +# CKPT_RESUME: set to 1 to resume training (keep iteration, optimizer, rng); +# default 0 = finetune mode (reset iteration, skip optim/rng) +CKPT_LOAD=${CKPT_LOAD:-} +CKPT_FORMAT=${CKPT_FORMAT:-} +CKPT_RESUME=${CKPT_RESUME:-0} +CKPT_OVERRIDE_SCHEDULER=${CKPT_OVERRIDE_SCHEDULER:-0} +CKPT_ARGS=() +if [ -n "$CKPT_LOAD" ]; then + CKPT_ARGS+=( --load "$CKPT_LOAD" ) + if [ "$CKPT_RESUME" -eq 0 ]; then + CKPT_ARGS+=( --finetune --no-load-optim --no-load-rng ) + fi + if [ -n "$CKPT_FORMAT" ]; then + CKPT_ARGS+=( --ckpt-format "$CKPT_FORMAT" ) + fi + if [ "$CKPT_OVERRIDE_SCHEDULER" -eq 1 ]; then + CKPT_ARGS+=( --override-opt-param-scheduler ) + fi +fi + +# --- FSDP --- +USE_FSDP=${USE_FSDP:-1} +if [ "$USE_FSDP" -eq 1 ]; then + FSDP_ARGS=( + --use-megatron-fsdp + --data-parallel-sharding-strategy optim_grads_params + --init-model-with-meta-device + --use-distributed-optimizer + --ckpt-format fsdp_dtensor + ) + export CUDA_DEVICE_MAX_CONNECTIONS=8 +else + FSDP_ARGS=() +fi + +echo "================================================================" +echo "Qwen3.5-VL Multimodal Training (multimodal_dev)" +echo " Variant: $MODEL_VARIANT" +echo " Vision layers: $VISION_NUM_LAYERS" +echo " GPUs per node: $GPUS_PER_NODE" +echo " Num nodes: $NUM_NODES" +echo " TP=$TP EP=$EP PP=$PP CP=$CP" +echo " MBS=$MBS GBS=$GBS" +echo " Launcher: $LAUNCHER" +echo " FSDP: $USE_FSDP" +echo " PROFILE: $PROFILE" +if [ -n "$CKPT_LOAD" ]; then + echo " CKPT_LOAD: $CKPT_LOAD" + echo " CKPT_FORMAT: ${CKPT_FORMAT:-auto}" + echo " CKPT_RESUME: $CKPT_RESUME" +fi +if [ "$PROFILE" = "1" ]; then + echo " Profile steps: ${PROFILE_STEP_START}-${PROFILE_STEP_END}" + echo " Profile ranks: $PROFILE_RANKS" +fi +echo "================================================================" + +if [ "$LAUNCHER" = "python" ]; then + LAUNCH_CMD=( python $MEGATRON_LM_PATH/examples/multimodal_dev/pretrain_multimodal.py ) +elif [ "$LAUNCHER" = "torchrun" ]; then + LAUNCH_CMD=( torchrun "${DISTRIBUTED_ARGS[@]}" $MEGATRON_LM_PATH/examples/multimodal_dev/pretrain_multimodal.py ) +else + echo "Unsupported LAUNCHER=$LAUNCHER (expected torchrun or python)" >&2 + exit 1 +fi + +cmd=( "${NSYS_CMD[@]}" "${LAUNCH_CMD[@]}" \ + "${TRAINING_ARGS[@]}" \ + "${PROFILE_ARGS[@]}" \ + "${MODEL_PARALLEL_ARGS[@]}" \ + "${EVAL_AND_LOGGING_ARGS[@]}" \ + "${TOKENIZER_ARGS[@]}" \ + "${MULTIMODAL_ARGS[@]}" \ + "${GPT_MODEL_ARGS[@]}" \ + "${MOE_ARGS[@]}" \ + "${RECOMPUTE_ARGS[@]}" \ + "${FSDP_ARGS[@]}" \ + "${CKPT_ARGS[@]}" ) + +echo "${cmd[@]}" + +if [ "$DRY_RUN" -eq 1 ]; then + echo "=== DRY RUN ===" + exit 0 +else + "${cmd[@]}" +fi diff --git a/examples/multimodal_dev/tests/__init__.py b/examples/multimodal_dev/tests/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/tests/_helpers.py b/examples/multimodal_dev/tests/_helpers.py new file mode 100644 index 00000000000..b0c69207f3a --- /dev/null +++ b/examples/multimodal_dev/tests/_helpers.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Test helpers shared across the multimodal_dev test suite.""" + + +def grad_norm(model): + """L2 norm of all populated parameter gradients on this rank.""" + total = 0.0 + for p in model.parameters(): + if p.grad is not None: + total += p.grad.data.float().norm(2).item() ** 2 + return total**0.5 + + +def mean_loss(per_token_loss, loss_mask): + """Mean per-token loss over valid (mask>0) positions on this rank.""" + flat = per_token_loss.float().view(-1) + mask = loss_mask.float().view(-1) + return (flat * mask).sum() / mask.sum().clamp(min=1) diff --git a/examples/multimodal_dev/tests/test_cp_correctness.py b/examples/multimodal_dev/tests/test_cp_correctness.py new file mode 100644 index 00000000000..fe156ed54af --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_correctness.py @@ -0,0 +1,313 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Distributed correctness test for Context Parallelism (CP) support. + +Verifies that CP>1 produces the same (or numerically close) loss as CP=1 +for the Qwen3.5-VL multimodal model by running forward passes with +deterministic data and comparing the per-rank reduced losses. + +Launch with torchrun (N must be >= 2*max_cp_size for zigzag splitting): + + # Test CP=2 on 2 GPUs: + torchrun --nproc_per_node=2 examples/multimodal_dev/tests/test_cp_correctness.py --cp-size 2 + + # Test CP=4 on 4 GPUs: + torchrun --nproc_per_node=4 examples/multimodal_dev/tests/test_cp_correctness.py --cp-size 4 + +The test: + 1. Builds a tiny proxy model (2 layers, no MoE, no vision encoder). + 2. Generates a deterministic batch (same seed on all ranks). + 3. Runs forward with CP=1 (each rank processes the full sequence independently). + 4. Re-initialises model-parallel groups with the target CP size. + 5. Runs forward with CP=target (sequence is split across ranks). + 6. Compares the all-reduced loss values. + +Exit code 0 = PASS, 1 = FAIL. +""" + +import argparse +import os +import sys + +import torch +import torch.distributed as dist + +# Ensure the repo root is on the path so that megatron and examples are importable. +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + + +def _parse_args(): + parser = argparse.ArgumentParser(description="CP correctness test") + parser.add_argument( + "--cp-size", type=int, default=2, + help="Target context-parallel size to compare against CP=1 baseline", + ) + parser.add_argument( + "--seq-len", type=int, default=128, + help="Sequence length (must be divisible by 2*max(cp_size, tp_size*cp_size))", + ) + parser.add_argument( + "--atol", type=float, default=1e-4, + help="Absolute tolerance for loss comparison", + ) + parser.add_argument( + "--rtol", type=float, default=5e-2, + help="Relative tolerance for loss comparison (default 5%%)", + ) + parser.add_argument( + "--seed", type=int, default=42, + help="Random seed for reproducibility", + ) + # Megatron adds extra args; ignore them. + args, _ = parser.parse_known_args() + return args + + +def _init_distributed(): + """Initialise torch.distributed if not already done.""" + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + return local_rank + + +def _init_megatron_parallel(tp_size=1, pp_size=1, cp_size=1, seed=42): + """(Re-)initialise Megatron model-parallel groups and RNG tracker.""" + from megatron.core import parallel_state as ps + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + ps.destroy_model_parallel() + ps.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + context_parallel_size=cp_size, + ) + model_parallel_cuda_manual_seed(seed) + + +def _make_deterministic_batch(seed, batch_size, seq_len, vocab_size, device): + """Create a deterministic batch identical on all ranks.""" + rng = torch.Generator(device="cpu") + rng.manual_seed(seed) + + input_ids = torch.randint( + 0, vocab_size, (batch_size, seq_len), generator=rng, + ).to(device) + labels = torch.randint( + 0, vocab_size, (batch_size, seq_len), generator=rng, + ).to(device) + loss_mask = torch.ones(batch_size, seq_len, device=device) + # Standard position_ids [B, S] + position_ids = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1) + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "position_ids": position_ids, + } + + +def _build_tiny_model(cp_size, device): + """Build a minimal GPTModel for testing (no vision, no MoE).""" + from megatron.core.models.gpt import GPTModel + from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec + from megatron.core.transformer.spec_utils import ModuleSpec + from megatron.core.transformer.transformer_config import TransformerConfig + + hidden_size = 256 + num_heads = 4 + config = TransformerConfig( + num_layers=2, + hidden_size=hidden_size, + ffn_hidden_size=hidden_size * 4, + num_attention_heads=num_heads, + kv_channels=hidden_size // num_heads, + normalization="RMSNorm", + layernorm_epsilon=1e-6, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + bf16=True, + context_parallel_size=cp_size, + add_bias_linear=False, + attention_dropout=0.0, + hidden_dropout=0.0, + sequence_parallel=False, + ) + + spec = get_gpt_layer_with_transformer_engine_spec() + + model = GPTModel( + config=config, + transformer_layer_spec=spec, + vocab_size=1024, + max_sequence_length=4096, + pre_process=True, + post_process=True, + parallel_output=False, + share_embeddings_and_output_weights=True, + position_embedding_type="rope", + rotary_percent=1.0, + rotary_base=10000, + ) + model = model.to(device=device, dtype=torch.bfloat16) + return model, config + + +def _forward_with_cp(model, batch, cp_size): + """Run forward pass, handling CP splitting of the batch. + + When cp_size > 1, splits the batch tensors using the same zigzag + logic as multimodal_dev/models/base.py. + """ + from examples.multimodal_dev.models.base import _cp_split_tensor + from megatron.core import parallel_state as ps + + input_ids = batch["input_ids"].clone() + labels = batch["labels"].clone() + loss_mask = batch["loss_mask"].clone() + position_ids = batch["position_ids"].clone() + + if cp_size > 1: + cp_rank = ps.get_context_parallel_rank() + input_ids = _cp_split_tensor(input_ids, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + labels = _cp_split_tensor(labels, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + loss_mask = _cp_split_tensor(loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + # position_ids are NOT split — the RoPE layer handles CP slicing internally. + + with torch.no_grad(): + output = model( + input_ids=input_ids, + position_ids=position_ids, + labels=labels, + attention_mask=None, + ) + + # output is the per-token loss [B, S/CP] + masked_loss = (output.float() * loss_mask.float()).sum() + num_tokens = loss_mask.sum() + + # All-reduce across CP ranks to get global loss + if cp_size > 1: + cp_group = ps.get_context_parallel_group() + dist.all_reduce(masked_loss, group=cp_group) + dist.all_reduce(num_tokens, group=cp_group) + + avg_loss = masked_loss / num_tokens.clamp(min=1) + return avg_loss.item() + + +def main(): + args = _parse_args() + local_rank = _init_distributed() + device = torch.device(f"cuda:{local_rank}") + world_size = dist.get_world_size() + rank = dist.get_rank() + + target_cp = args.cp_size + if world_size < target_cp: + if rank == 0: + print( + f"SKIP: world_size={world_size} < cp_size={target_cp}. " + f"Need at least {target_cp} GPUs.", + flush=True, + ) + dist.destroy_process_group() + sys.exit(0) + if world_size % target_cp != 0: + if rank == 0: + print( + f"SKIP: world_size={world_size} is not divisible by cp_size={target_cp}.", + flush=True, + ) + dist.destroy_process_group() + sys.exit(0) + + vocab_size = 1024 + + # Ensure seq_len is divisible by 2 * target_cp + seq_len = args.seq_len + align = 2 * target_cp + if seq_len % align != 0: + seq_len = ((seq_len + align - 1) // align) * align + if rank == 0: + print(f"Adjusted seq_len to {seq_len} for alignment with CP={target_cp}", flush=True) + + # --- Step 1: CP=1 baseline --- + if rank == 0: + print(f"=== CP=1 baseline (world_size={world_size}) ===", flush=True) + + _init_megatron_parallel(cp_size=1) + + # Set deterministic seed for model init + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + model_cp1, _ = _build_tiny_model(cp_size=1, device=device) + + batch = _make_deterministic_batch( + seed=args.seed + 1, batch_size=1, seq_len=seq_len, + vocab_size=vocab_size, device=device, + ) + + loss_cp1 = _forward_with_cp(model_cp1, batch, cp_size=1) + + if rank == 0: + print(f" CP=1 loss: {loss_cp1:.6f}", flush=True) + + # Save model state for reuse + state_dict = model_cp1.state_dict() + del model_cp1 + torch.cuda.empty_cache() + + # --- Step 2: CP=target --- + if rank == 0: + print(f"=== CP={target_cp} (world_size={world_size}) ===", flush=True) + + _init_megatron_parallel(cp_size=target_cp) + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + model_cpN, _ = _build_tiny_model(cp_size=target_cp, device=device) + + # Load the same weights to ensure identical model + model_cpN.load_state_dict(state_dict, strict=True) + del state_dict + + loss_cpN = _forward_with_cp(model_cpN, batch, cp_size=target_cp) + + if rank == 0: + print(f" CP={target_cp} loss: {loss_cpN:.6f}", flush=True) + + del model_cpN + torch.cuda.empty_cache() + + # --- Step 3: Compare --- + if rank == 0: + diff = abs(loss_cpN - loss_cp1) + rel_diff = diff / max(abs(loss_cp1), 1e-10) + + print(f"\n=== Comparison ===", flush=True) + print(f" CP=1 loss: {loss_cp1:.6f}", flush=True) + print(f" CP={target_cp} loss: {loss_cpN:.6f}", flush=True) + print(f" Absolute diff: {diff:.6e}", flush=True) + print(f" Relative diff: {rel_diff:.6e}", flush=True) + print(f" Tolerance (atol): {args.atol:.6e}", flush=True) + print(f" Tolerance (rtol): {args.rtol:.6e}", flush=True) + + passed = diff <= args.atol + args.rtol * abs(loss_cp1) + if passed: + print(f"\nPASS: CP={target_cp} matches CP=1 baseline", flush=True) + else: + print(f"\nFAIL: CP={target_cp} loss differs from CP=1 beyond tolerance", flush=True) + + dist.barrier() + dist.destroy_process_group() + + if rank == 0 and not passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_cp_support.py b/examples/multimodal_dev/tests/test_cp_support.py new file mode 100644 index 00000000000..d46b5d8ef71 --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_support.py @@ -0,0 +1,347 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Unit tests for Context Parallelism (CP) support in multimodal_dev. + +Tests cover: + 1. _cp_split_tensor — zigzag split correctness, reconstruction, and edge cases + 2. _NoCPGroup — dummy process group behaviour + 3. _thd_cp_partition_index — TE-based per-sample THD CP partitioning + 4. Cross-validation against megatron.core.utils.get_batch_on_this_cp_rank + +Run with: pytest examples/multimodal_dev/tests/test_cp_support.py -v +""" + +import pytest +import torch + +from examples.multimodal_dev.models.base import _cp_split_tensor, _NoCPGroup + + +class TestCpSplitTensor: + """Tests for zigzag CP splitting.""" + + def test_basic_2d_cp2(self): + """[B, S] tensor with CP=2 splits and reconstructs correctly.""" + B, S = 2, 16 + t = torch.arange(B * S).reshape(B, S) + cp_size = 2 + + chunks = [] + for rank in range(cp_size): + chunks.append(_cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank)) + + # Each rank gets S / CP = 8 tokens + for c in chunks: + assert c.shape == (B, S // cp_size) + + # Reconstruct: rank 0 gets chunks [0, 3], rank 1 gets chunks [1, 2] + # Original split into 4 chunks of size 4: + # chunk0=[0..3], chunk1=[4..7], chunk2=[8..11], chunk3=[12..15] + # rank0 = [chunk0, chunk3] = [0..3, 12..15] + # rank1 = [chunk1, chunk2] = [4..7, 8..11] + assert torch.equal(chunks[0][0], torch.tensor([0, 1, 2, 3, 12, 13, 14, 15])) + assert torch.equal(chunks[1][0], torch.tensor([4, 5, 6, 7, 8, 9, 10, 11])) + + def test_3d_mrope_cp2(self): + """[3, B, S] MRoPE tensor with CP=2.""" + B, S = 1, 8 + cp_size = 2 + t = torch.arange(3 * B * S).reshape(3, B, S) + + chunk = _cp_split_tensor(t, seq_dim=2, cp_size=cp_size, cp_rank=0) + assert chunk.shape == (3, B, S // cp_size) + + # All 3 MRoPE components should be split consistently + for d in range(3): + original_row = t[d, 0] # [S] + # With S=8, CP=2: 4 chunks of size 2 + # rank0 gets chunks [0, 3] = positions [0,1, 6,7] + expected = torch.cat([original_row[0:2], original_row[6:8]]) + assert torch.equal(chunk[d, 0], expected) + + def test_sbh_decoder_input(self): + """[S, B, H] decoder input split along dim=0.""" + S, B, H = 16, 2, 4 + cp_size = 2 + t = torch.randn(S, B, H) + + chunk = _cp_split_tensor(t, seq_dim=0, cp_size=cp_size, cp_rank=0) + assert chunk.shape == (S // cp_size, B, H) + + def test_cp4(self): + """CP=4 zigzag pattern.""" + S = 32 + cp_size = 4 + t = torch.arange(S).unsqueeze(0) # [1, 32] + + all_chunks = [] + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + all_chunks.append(c) + assert c.shape == (1, S // cp_size) + + # All tokens should appear exactly once across ranks + combined = torch.cat(all_chunks, dim=1) + assert torch.equal(combined.sort(dim=1).values, t.sort(dim=1).values) + + def test_cp8(self): + """CP=8 zigzag pattern — all tokens appear exactly once.""" + S = 64 + cp_size = 8 + t = torch.arange(S).unsqueeze(0) # [1, 64] + + all_chunks = [] + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + all_chunks.append(c) + assert c.shape == (1, S // cp_size) + + combined = torch.cat(all_chunks, dim=1) + assert torch.equal(combined.sort(dim=1).values, t.sort(dim=1).values) + + def test_not_divisible_raises(self): + """Should raise when seq_len not divisible by 2*cp_size.""" + t = torch.randn(2, 10) # S=10, not divisible by 4 + with pytest.raises(AssertionError): + _cp_split_tensor(t, seq_dim=1, cp_size=2, cp_rank=0) + + def test_zigzag_symmetry(self): + """rank 0 and rank (cp_size-1) should get mirror chunks.""" + S = 16 + cp_size = 2 + t = torch.arange(S).unsqueeze(0) # [1, 16] + + c0 = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=0) + c1 = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=1) + + # rank0 gets chunks [0, 3], rank1 gets chunks [1, 2] + # chunk0=[0..3], chunk3=[12..15] -> rank0 gets [0..3, 12..15] + # chunk1=[4..7], chunk2=[8..11] -> rank1 gets [4..7, 8..11] + # rank0's first half is earliest, rank1's first half is next + assert c0[0, 0].item() < c1[0, 0].item() # rank0 starts earlier + + def test_matches_megatron_core(self): + """Cross-validate against megatron.core.utils.get_batch_on_this_cp_rank logic. + + We simulate the core function's logic (seq_dim=1, attention_mask seq_dim=2) + and compare. + """ + B, S = 2, 32 + cp_size = 4 + + input_ids = torch.arange(B * S).reshape(B, S) + labels = torch.arange(B * S).reshape(B, S) + 1000 + + for cp_rank in range(cp_size): + # Our implementation + our_ids = _cp_split_tensor(input_ids, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + our_labels = _cp_split_tensor(labels, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + + # Simulate megatron core logic inline + def core_split(val, seq_dim): + val = val.view( + *val.shape[0:seq_dim], + 2 * cp_size, + val.shape[seq_dim] // (2 * cp_size), + *val.shape[(seq_dim + 1):], + ) + index = torch.zeros(2, dtype=torch.int64, device=val.device) + index[0].fill_(cp_rank) + index[1].fill_(2 * cp_size - cp_rank - 1) + val = val.index_select(seq_dim, index) + val = val.view(*val.shape[0:seq_dim], -1, *val.shape[(seq_dim + 2):]) + return val + + ref_ids = core_split(input_ids.clone(), seq_dim=1) + ref_labels = core_split(labels.clone(), seq_dim=1) + + assert torch.equal(our_ids, ref_ids), f"input_ids mismatch at rank {cp_rank}" + assert torch.equal(our_labels, ref_labels), f"labels mismatch at rank {cp_rank}" + + def test_batch_dim_preserved(self): + """Batch dimension must be unchanged after split.""" + B, S = 4, 32 + cp_size = 4 + t = torch.randn(B, S) + + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + assert c.shape[0] == B + + +class TestNoCPGroup: + """Tests for the dummy CP group used by the vision encoder.""" + + def test_size_is_one(self): + g = _NoCPGroup() + assert g.size() == 1 + + def test_rank_is_zero(self): + g = _NoCPGroup() + assert g.rank() == 0 + + +try: + from transformer_engine.pytorch import cpp_extensions as _tex # noqa: F401 + + _HAS_TE = True +except Exception: + _HAS_TE = False + + +@pytest.mark.skipif(not _HAS_TE, reason="TransformerEngine not installed") +class TestThdCpPartition: + """Verify TE-based per-sample THD + CP partition matches THD semantics. + + Each packed sub-sample of length ``s_i`` (where ``s_i % (2*cp_size) == 0``) + is split into ``2*cp_size`` zigzag chunks per sample; rank ``r`` gets + chunks ``[r, 2*cp_size - r - 1]`` of every sample. The union across + ranks must cover every token position exactly once. + """ + + @staticmethod + def _make_padded_packed(seqlens, divisor): + """Concatenate per-sample dummy tokens after padding each sample to a + multiple of *divisor*. Returns ``(input_ids[1, T], cu_seqlens_padded)``. + """ + import math + padded = [math.ceil(s / divisor) * divisor for s in seqlens] + chunks = [] + next_id = 1 + for s, p in zip(seqlens, padded): + chunks.append(torch.arange(next_id, next_id + s, dtype=torch.int64)) + chunks.append(torch.zeros(p - s, dtype=torch.int64)) # padding + next_id += s + input_ids = torch.cat(chunks, dim=0).unsqueeze(0) # [1, T] + cu_seqlens_padded = torch.tensor( + [0] + list(torch.tensor(padded).cumsum(0).tolist()), + dtype=torch.int32, + ) + return input_ids, cu_seqlens_padded + + def _ensure_cuda(self, x): + return x.cuda() if torch.cuda.is_available() else x + + def test_partition_covers_all_positions_cp2(self): + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [5, 7, 3] # valid lengths + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + + # Union of per-rank indices must be all positions exactly once. + seen = torch.zeros(T, dtype=torch.long, device=input_ids.device) + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + assert idx.numel() == T // cp_size, ( + f"rank {cp_rank}: expected {T // cp_size} tokens, got {idx.numel()}" + ) + seen.scatter_add_( + 0, idx.long(), torch.ones_like(idx, dtype=seen.dtype), + ) + assert torch.all(seen == 1), ( + f"Position coverage broken: counts={seen.tolist()}" + ) + + def test_index_select_aligns_inputs_and_position_ids_cp2(self): + """input_ids, loss_mask, and (3, 1, T) position_ids index_select with + the same partition index produce shape-consistent per-rank tensors.""" + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [8, 4] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + labels = input_ids + 1000 + loss_mask = (input_ids != 0).float() + position_ids = ( + torch.arange(T, device=input_ids.device) + .unsqueeze(0).unsqueeze(0).expand(3, 1, T).contiguous() + ) + H = 4 + decoder_input = ( + torch.arange(T * H, dtype=torch.float32, device=input_ids.device) + .view(T, 1, H) + ) + + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + ii = input_ids.index_select(1, idx) + ll = labels.index_select(1, idx) + lm = loss_mask.index_select(1, idx) + pi = position_ids.index_select(2, idx) + di = decoder_input.index_select(0, idx) + + assert ii.shape == (1, T // cp_size) + assert ll.shape == (1, T // cp_size) + assert lm.shape == (1, T // cp_size) + assert pi.shape == (3, 1, T // cp_size) + assert di.shape == (T // cp_size, 1, H) + # Sliced position_ids is just the partition index itself + # (since position_ids was arange(T) over all positions). + assert torch.equal(pi[0, 0], idx.to(pi.dtype)) + # All MRoPE rows agree. + assert torch.equal(pi[1, 0], pi[0, 0]) + assert torch.equal(pi[2, 0], pi[0, 0]) + + def test_partition_cp4_three_samples(self): + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 4 + seqlens = [12, 4, 8] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + + seen = torch.zeros(T, dtype=torch.long, device=input_ids.device) + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + assert idx.numel() == T // cp_size + seen.scatter_add_( + 0, idx.long(), torch.ones_like(idx, dtype=seen.dtype), + ) + assert torch.all(seen == 1) + + def test_loss_mask_zero_kept_per_rank(self): + """Pad-token positions (loss_mask=0) survive as 0 on whichever rank + they land — sanity check that we don't accidentally discard them.""" + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [5, 3] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + loss_mask = (input_ids != 0).float() + total_zeros = (loss_mask == 0).sum().item() + + zeros_seen = 0 + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + zeros_seen += ( + loss_mask.index_select(1, idx) == 0 + ).sum().item() + assert zeros_seen == total_zeros diff --git a/examples/multimodal_dev/tests/test_cp_thd_correctness.py b/examples/multimodal_dev/tests/test_cp_thd_correctness.py new file mode 100644 index 00000000000..e815a948474 --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_thd_correctness.py @@ -0,0 +1,455 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# This is a stdout-reporting standalone script; `print` is intentional. +# pylint: disable=bad-builtin + +"""CP=1 vs CP=4 correctness test for THD and BSHD packing. + +Runs the production forward path (:class:`MultimodalModel`) twice in a +single ``torchrun`` invocation: + + Phase 1 — CP=1 baseline. All 4 ranks initialise with TP=1, CP=1 + (DP=4 implicit). Each rank computes the full sequence, + producing identical loss / grad_norm on every rank; rank 0's + value is the baseline. + + Phase 2 — CP=4. After ``destroy_model_parallel`` + ``initialize_model_parallel(CP=4)`` + the 4 ranks form a single CP group. The model's internal + ``_cp_split_for_forward`` slices inputs per rank; per-rank + loss / gradients are aggregated via AllReduce on the CP group. + +We compare CP=1 and CP=4 results for both BSHD and THD packing modes, +asserting that loss and grad_norm match within tolerance. + +Run with:: + + PYTHONPATH=. torchrun --nproc-per-node 4 \\ + examples/multimodal_dev/tests/test_cp_thd_correctness.py +""" + +import argparse +import os +import sys + +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import pack_or_pad_batch +from examples.multimodal_dev.models.base import ( + MultimodalModel, + _cp_split_tensor, + _thd_cp_partition_index, +) +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.parallel_state import get_context_parallel_group, get_context_parallel_rank +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.transformer_config import TransformerConfig +from tests.unit_tests.test_utilities import Utils + +# =================================================================== +# Stub vision encoder +# =================================================================== + + +class _StubVisionEncoder(MegatronModule): + """Vision encoder placeholder. The vision branch is skipped in + :meth:`MultimodalModel.forward` whenever ``pixel_values is None``, so + this module is never called — it only satisfies the constructor's + ``vision_encoder: MegatronModule`` requirement. + """ + + def __init__(self, config): + """Initialise the stub with the given TransformerConfig.""" + super().__init__(config=config) + + def forward(self, pixel_values, image_grid_thw): + """Never called when ``pixel_values=None``; raises if it ever is.""" + raise RuntimeError("vision branch should not run when pixel_values=None") + + +# =================================================================== +# Model builder +# =================================================================== + + +def _build_model(config, vocab_size, max_seq_len, image_token_id): + spec = get_gpt_layer_with_transformer_engine_spec() + vision = _StubVisionEncoder(config) + model = MultimodalModel( + language_config=config, + language_spec=spec, + vision_encoder=vision, + vocab_size=vocab_size, + max_sequence_length=max_seq_len, + image_token_id=image_token_id, + position_embedding_type="rope", + parallel_output=False, + ) + model.cuda() + return model + + +def _make_config( + num_layers, hidden_size, ffn_hidden_size, num_heads, num_kv_heads, context_parallel_size +): + return TransformerConfig( + num_layers=num_layers, + hidden_size=hidden_size, + ffn_hidden_size=ffn_hidden_size, + num_attention_heads=num_heads, + num_query_groups=num_kv_heads, + bf16=True, + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + hidden_dropout=0.0, + attention_dropout=0.0, + tensor_model_parallel_size=1, + context_parallel_size=context_parallel_size, + sequence_parallel=False, + ) + + +# =================================================================== +# Loss / grad-norm aggregation +# =================================================================== + + +def _global_loss(output, rank_loss_mask, cp_size): + """Mean per-token loss over all CP shards (matches CP=1 mean exactly).""" + num = (output.float().view(-1) * rank_loss_mask.float().view(-1)).sum() + den = rank_loss_mask.float().view(-1).sum().clamp(min=1) + if cp_size > 1: + group = get_context_parallel_group() + torch.distributed.all_reduce(num, group=group) + torch.distributed.all_reduce(den, group=group) + return (num / den).item() + + +def _global_grad_norm(model, cp_size): + """Global L2 grad norm. For CP>1, AllReduce(SUM) gradients across CP + then divide by ``cp_size`` so each rank holds the CP-mean gradient + (matching CP=1's behaviour, where backward on the per-batch mean loss + yields exactly that gradient). + """ + if cp_size > 1: + group = get_context_parallel_group() + for p in model.parameters(): + if p.grad is not None: + torch.distributed.all_reduce(p.grad, group=group) + p.grad /= cp_size + + sq = 0.0 + for p in model.parameters(): + if p.grad is not None: + sq += p.grad.float().norm(2).item() ** 2 + return sq**0.5 + + +# =================================================================== +# Data — identical across all ranks (deterministic generator) +# =================================================================== + + +def _make_data(B, S, vocab_size, image_token_id, seed): + """Same input on every rank thanks to the seeded generator.""" + g = torch.Generator(device="cuda") + g.manual_seed(seed) + input_ids = torch.randint(0, vocab_size, (B, S), generator=g, device="cuda") + # Ensure no accidental image tokens (we never run the vision branch). + input_ids = torch.where(input_ids == image_token_id, (input_ids + 1) % vocab_size, input_ids) + labels = torch.randint(0, vocab_size, (B, S), generator=g, device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + position_ids = torch.arange(S, device="cuda").unsqueeze(0).expand(B, -1).contiguous() + return input_ids, labels, loss_mask, position_ids + + +# =================================================================== +# One BSHD or THD forward+backward, returning (loss, grad_norm) +# =================================================================== + + +def _run_bshd(model, B, S, vocab_size, image_token_id, cp_size, seed): + input_ids, labels, loss_mask, position_ids = _make_data(B, S, vocab_size, image_token_id, seed) + + output = model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + labels=labels, + loss_mask=loss_mask, + pixel_values=None, + image_grid_thw=None, + packed_seq_params=None, + ) + + # Slice loss_mask the same way forward_step does for BSHD + CP. + rank_loss_mask = loss_mask + if cp_size > 1: + rank_loss_mask = _cp_split_tensor( + rank_loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=get_context_parallel_rank() + ) + + loss_val = _global_loss(output, rank_loss_mask, cp_size) + + # Backward on the LOCAL mean loss (each rank's contribution + # equal-weighted; SUM-then-divide across CP recovers CP=1's gradient). + local = ( + output.float().view(-1) * rank_loss_mask.float().view(-1) + ).sum() / rank_loss_mask.float().view(-1).sum().clamp(min=1) + model.zero_grad() + local.backward() + + gn = _global_grad_norm(model, cp_size) + return loss_val, gn + + +def _run_thd(model, B, S, vocab_size, image_token_id, cp_size, seed): + input_ids, labels, loss_mask, _ = _make_data(B, S, vocab_size, image_token_id, seed) + + # Build the per-sample dict list and pack to [1, T]. + samples = [] + for i in range(B): + samples.append( + { + "input_ids": input_ids[i].clone(), + "labels": labels[i].clone(), + "loss_mask": loss_mask[i].clone(), + # No vision; empty tensors satisfy pack_or_pad_batch. + "pixel_values": torch.zeros(0, 1, device="cuda"), + "image_grid_thw": torch.empty(0, 3, dtype=torch.long, device="cuda"), + } + ) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + + # THD position_ids: per-sample restart at 0. Each sample has length S + # (equal-length data), so this is arange(S) repeated B times. + thd_pos = ( + torch.cat([torch.arange(S, device="cuda") for _ in range(B)]).unsqueeze(0).contiguous() + ) + + output = model( + input_ids=packed["input_ids"], + position_ids=thd_pos, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + pixel_values=None, + image_grid_thw=None, + packed_seq_params=psp, + ) + + rank_loss_mask = packed["loss_mask"] + if cp_size > 1: + T = rank_loss_mask.shape[1] + idx = _thd_cp_partition_index( + psp.cu_seqlens_q_padded, T, cp_size, get_context_parallel_rank() + ) + rank_loss_mask = rank_loss_mask.index_select(1, idx) + + loss_val = _global_loss(output, rank_loss_mask, cp_size) + + local = ( + output.float().view(-1) * rank_loss_mask.float().view(-1) + ).sum() / rank_loss_mask.float().view(-1).sum().clamp(min=1) + model.zero_grad() + local.backward() + + gn = _global_grad_norm(model, cp_size) + return loss_val, gn + + +# =================================================================== +# State-dict roundtrip — keep weights identical across phases +# =================================================================== + + +def _cpu_state_dict(model): + """Snapshot of model.state_dict() detached to CPU (kept in memory). + + Some entries (TransformerEngine ``_extra_state``) are non-tensor or + ``None``; pass them through untouched. + """ + snap = {} + for k, v in model.state_dict().items(): + if isinstance(v, torch.Tensor): + snap[k] = v.detach().to("cpu").clone() + else: + snap[k] = v + return snap + + +def _restore_state_dict(model, snapshot): + """Load a saved snapshot back into a freshly built model.""" + payload = {k: (v.to("cuda") if isinstance(v, torch.Tensor) else v) for k, v in snapshot.items()} + model.load_state_dict(payload) + + +# =================================================================== +# Main +# =================================================================== + + +def _is_rank0(): + return not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + + +def _print_banner(title): + if _is_rank0(): + print(f"\n{'=' * 60}") + print(f" {title}") + print(f"{'=' * 60}") + + +def _print_compare(label, baseline, trial, atol, rtol): + """Print a CP=1 vs CP=4 comparison line and return whether it passed.""" + if not _is_rank0(): + return True + + abs_diff = abs(baseline - trial) + rel_diff = abs_diff / max(abs(baseline), 1e-8) + ok = abs_diff < atol or rel_diff < rtol + flag = "PASS" if ok else "FAIL" + print( + f" {label:<30s} CP=1: {baseline:.8f} CP=4: {trial:.8f}" + f" abs={abs_diff:.2e} rel={rel_diff:.2e} [{flag}]" + ) + return ok + + +def main(): + """Run CP=1 baseline + CP=4 trial and compare losses / grad_norms.""" + parser = argparse.ArgumentParser() + parser.add_argument("--batch-size", type=int, default=2) + # Must be divisible by 2*cp_size (=8 for CP=4 zigzag). + parser.add_argument("--seq-len", type=int, default=64) + parser.add_argument("--vocab-size", type=int, default=1024) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--num-layers", type=int, default=2) + parser.add_argument("--num-heads", type=int, default=4) + parser.add_argument("--num-kv-heads", type=int, default=2) + parser.add_argument("--ffn-hidden-size", type=int, default=512) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--atol-loss", type=float, default=1e-3) + parser.add_argument("--rtol-grad", type=float, default=5e-3) + parser.add_argument("--data-seed", type=int, default=123) + args = parser.parse_args() + + image_token_id = 0 # never appears in input (data filters this id out) + + # ---------------------------------------------------------------- + # Phase 1: CP=1 baseline + # ---------------------------------------------------------------- + _print_banner("Phase 1 — building CP=1 baseline (TP=1, CP=1, DP=4)") + Utils.initialize_model_parallel(tensor_model_parallel_size=1, context_parallel_size=1) + model_parallel_cuda_manual_seed(args.seed) + + config_cp1 = _make_config( + args.num_layers, + args.hidden_size, + args.ffn_hidden_size, + args.num_heads, + args.num_kv_heads, + context_parallel_size=1, + ) + torch.manual_seed(args.seed) + model_cp1 = _build_model(config_cp1, args.vocab_size, args.seq_len, image_token_id) + + bshd_loss_cp1, bshd_gn_cp1 = _run_bshd( + model_cp1, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=1, + seed=args.data_seed, + ) + thd_loss_cp1, thd_gn_cp1 = _run_thd( + model_cp1, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=1, + seed=args.data_seed, + ) + + # Snapshot weights *before* the optimizer would have touched them. + # (We've zeroed grads but never stepped; weights at this point are + # the just-initialised baseline.) + weights_snapshot = _cpu_state_dict(model_cp1) + del model_cp1 + torch.cuda.empty_cache() + + # ---------------------------------------------------------------- + # Phase 2: CP=4 + # ---------------------------------------------------------------- + _print_banner("Phase 2 — re-initialising for CP=4 (TP=1, CP=4)") + Utils.destroy_model_parallel() + Utils.initialize_model_parallel(tensor_model_parallel_size=1, context_parallel_size=4) + model_parallel_cuda_manual_seed(args.seed) + + config_cp4 = _make_config( + args.num_layers, + args.hidden_size, + args.ffn_hidden_size, + args.num_heads, + args.num_kv_heads, + context_parallel_size=4, + ) + torch.manual_seed(args.seed) + model_cp4 = _build_model(config_cp4, args.vocab_size, args.seq_len, image_token_id) + _restore_state_dict(model_cp4, weights_snapshot) + + bshd_loss_cp4, bshd_gn_cp4 = _run_bshd( + model_cp4, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=4, + seed=args.data_seed, + ) + thd_loss_cp4, thd_gn_cp4 = _run_thd( + model_cp4, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=4, + seed=args.data_seed, + ) + + # ---------------------------------------------------------------- + # Compare + # ---------------------------------------------------------------- + _print_banner("Results — CP=1 vs CP=4") + all_ok = True + all_ok &= _print_compare( + "BSHD loss", bshd_loss_cp1, bshd_loss_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "BSHD grad_norm", bshd_gn_cp1, bshd_gn_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "THD loss", thd_loss_cp1, thd_loss_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "THD grad_norm", thd_gn_cp1, thd_gn_cp4, args.atol_loss, args.rtol_grad + ) + + _print_banner("Summary") + if _is_rank0(): + print(f" {'ALL TESTS PASSED' if all_ok else 'SOME TESTS FAILED'}") + print(f"{'=' * 60}\n") + + Utils.destroy_model_parallel() + if not all_ok: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_mrope_parity.py b/examples/multimodal_dev/tests/test_mrope_parity.py new file mode 100644 index 00000000000..154284a4fc1 --- /dev/null +++ b/examples/multimodal_dev/tests/test_mrope_parity.py @@ -0,0 +1,663 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Parity tests for ``get_rope_index`` (MRoPE position-ID computation). + +Two properties are verified: + +1. **BSHD backwards compatibility** — the refactored ``get_rope_index`` + returns bit-identical ``(position_ids, mrope_position_deltas)`` to + the pre-refactor implementation on padded ``[B, S]`` batches. +2. **THD == BSHD on the valid region** — when the same variable-length + samples are fed through both layouts (BSHD with right-padding; THD + packed with ``cu_seqlens_q`` / ``cu_seqlens_q_padded``), positions at + every valid slot agree. + +The pre-refactor function is pinned inline as ``_old_get_rope_index`` +so this test stays self-contained. Run with:: + + python -m pytest examples/multimodal_dev/tests/test_mrope_parity.py -v + +or directly:: + + python examples/multimodal_dev/tests/test_mrope_parity.py +""" + +import math +import os +import sys +from itertools import accumulate + +import torch +import torch.nn.functional as F + +_REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../.."), +) +# Insert at position 0 unconditionally — other entries on sys.path +# (e.g. a sibling Megatron-LM checkout) have their own ``examples`` +# package that would otherwise shadow ours. +if _REPO_ROOT in sys.path: + sys.path.remove(_REPO_ROOT) +sys.path.insert(0, _REPO_ROOT) + +from megatron.core.packed_seq_params import PackedSeqParams + +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index + +# ----------------------------------------------------------------------------- +# Token-ID constants (match Qwen3.5-VL, but values are arbitrary for this test) +# ----------------------------------------------------------------------------- + +IMAGE_TOKEN_ID = 248056 +VIDEO_TOKEN_ID = 248057 +VISION_START_TOKEN_ID = 248053 +SPATIAL_MERGE_SIZE = 2 + + +# ----------------------------------------------------------------------------- +# Pinned reference implementation (pre-refactor BSHD path) +# ----------------------------------------------------------------------------- + +def _old_get_rope_index( + spatial_merge_size, + image_token_id, + video_token_id, + vision_start_token_id, + input_ids=None, + image_grid_thw=None, + video_grid_thw=None, + attention_mask=None, +): + """Pre-refactor BSHD implementation of ``get_rope_index``. + + Copied verbatim (modulo the broken cu_seqlens branch, which this + parity test does not exercise) so we can diff against the new + implementation on BSHD inputs. + """ + if video_grid_thw is not None: + video_grid_thw = torch.repeat_interleave( + video_grid_thw, video_grid_thw[:, 0], dim=0, + ) + video_grid_thw[:, 0] = 1 + + mrope_position_deltas = [] + + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + + for i, sample_input_ids in enumerate(total_input_ids): + sample_input_ids = sample_input_ids[attention_mask[i] == 1] + vision_start_indices = torch.argwhere( + sample_input_ids == vision_start_token_id, + ).squeeze(1) + vision_tokens = sample_input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = sample_input_ids.tolist() + llm_pos_ids_list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + + text_len + + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[ + ..., i, attention_mask[i] == 1 + ] = llm_positions.to(position_ids.device) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]), + ) + + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=total_input_ids.device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # Text-only fallback. + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = ( + position_ids.max(0, keepdim=False)[0] + .max(-1, keepdim=True)[0] + ) + mrope_position_deltas = ( + max_position_ids + 1 - attention_mask.shape[-1] + ) + else: + position_ids = ( + torch.arange( + input_ids.shape[1], device=input_ids.device, + ) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + return position_ids, mrope_position_deltas + + +# ----------------------------------------------------------------------------- +# Synthetic-sample builder +# ----------------------------------------------------------------------------- + +def _build_sample( + prefix_text_len, + grids, + suffix_text_len, + text_token=100, +): + """Build one variable-length sample with ``len(grids)`` images. + + Layout per image: ``vision_start_id`` then + ``llm_grid_t * llm_grid_h * llm_grid_w`` ``image_token_id`` slots + (where ``llm_grid_* = grid_* // spatial_merge_size`` for h/w). + Grids use ``t=1``. + + Returns ``(input_ids [L], image_grid_thw [N, 3])``. + """ + tokens = [text_token] * prefix_text_len + grid_rows = [] + for t, h, w in grids: + n_image_tokens = ( + t * (h // SPATIAL_MERGE_SIZE) * (w // SPATIAL_MERGE_SIZE) + ) + tokens.append(VISION_START_TOKEN_ID) + tokens.extend([IMAGE_TOKEN_ID] * n_image_tokens) + grid_rows.append([t, h, w]) + tokens.extend([text_token + 1] * suffix_text_len) + input_ids = torch.tensor(tokens, dtype=torch.int64) + image_grid_thw = torch.tensor(grid_rows, dtype=torch.int64) + return input_ids, image_grid_thw + + +def _sample_bank(): + """A small bank of samples covering text-only, single-image, multi-image.""" + return [ + _build_sample( + prefix_text_len=5, + grids=[(1, 4, 4)], + suffix_text_len=7, + ), + _build_sample( + prefix_text_len=3, + grids=[(1, 2, 2), (1, 4, 6)], + suffix_text_len=4, + ), + _build_sample( + prefix_text_len=10, + grids=[], + suffix_text_len=0, + ), + _build_sample( + prefix_text_len=0, + grids=[(1, 6, 4)], + suffix_text_len=2, + ), + ] + + +# ----------------------------------------------------------------------------- +# Test 1: BSHD backwards compatibility +# ----------------------------------------------------------------------------- + +def test_bshd_matches_old_reference(): + """New ``get_rope_index`` equals the pinned reference on BSHD inputs.""" + samples = _sample_bank() + max_len = max(s.numel() for s, _ in samples) + + input_ids_rows = [] + mask_rows = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + padded = F.pad(tokens, (0, max_len - L), value=0) + mask = torch.zeros(max_len, dtype=torch.int64) + mask[:L] = 1 + input_ids_rows.append(padded) + mask_rows.append(mask) + if grids.numel() > 0: + grid_rows.append(grids) + + input_ids = torch.stack(input_ids_rows) # [B, S] + attention_mask = torch.stack(mask_rows) # [B, S] + image_grid_thw = ( + torch.cat(grid_rows, dim=0) if grid_rows else None + ) + + old_pos, old_delta = _old_get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + new_pos, new_delta = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + packed_seq_params=None, + ) + + assert torch.equal(old_pos, new_pos), ( + f"BSHD position_ids differ.\nold:\n{old_pos}\nnew:\n{new_pos}" + ) + assert torch.equal(old_delta, new_delta), ( + f"BSHD mrope_position_deltas differ.\n" + f"old: {old_delta}\nnew: {new_delta}" + ) + + +# ----------------------------------------------------------------------------- +# Test 2: THD positions match BSHD positions on the valid region +# ----------------------------------------------------------------------------- + +def _pack_samples(samples, divisible_by=1): + """Pack ``samples`` into a single ``[1, T]`` tensor the same way + ``pack_or_pad_batch`` does, and build ``PackedSeqParams``. + + Each per-sample tensor is right-padded to a multiple of + ``divisible_by`` before concatenation. ``cu_seqlens_q`` tracks + unpadded lengths; ``cu_seqlens_q_padded`` tracks the packed layout. + """ + padded_chunks = [] + seqlens = [] + seqlens_padded = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + target_L = math.ceil(L / divisible_by) * divisible_by + padded_chunks.append(F.pad(tokens, (0, target_L - L), value=0)) + seqlens.append(L) + seqlens_padded.append(target_L) + if grids.numel() > 0: + grid_rows.append(grids) + + packed = torch.cat(padded_chunks, dim=0).unsqueeze(0) # [1, T] + cu_seqlens = torch.tensor( + list(accumulate(seqlens, initial=0)), dtype=torch.int32, + ) + cu_seqlens_padded = torch.tensor( + list(accumulate(seqlens_padded, initial=0)), dtype=torch.int32, + ) + psp = PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + max_seqlen_q=max(seqlens_padded), + max_seqlen_kv=max(seqlens_padded), + ) + image_grid_thw = ( + torch.cat(grid_rows, dim=0) if grid_rows else None + ) + return packed, psp, image_grid_thw, seqlens, seqlens_padded + + +def test_thd_matches_bshd_padded(): + """THD positions at every valid slot equal BSHD positions on the + equivalent right-padded batch. + """ + samples = _sample_bank() + + # BSHD side: right-pad to common max_len. + max_len = max(s.numel() for s, _ in samples) + input_ids_rows = [] + mask_rows = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + input_ids_rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + mask_rows.append(m) + if grids.numel() > 0: + grid_rows.append(grids) + bshd_input_ids = torch.stack(input_ids_rows) + bshd_mask = torch.stack(mask_rows) + bshd_grid = torch.cat(grid_rows, dim=0) if grid_rows else None + + bshd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=bshd_input_ids, + image_grid_thw=bshd_grid, + attention_mask=bshd_mask, + ) + # bshd_pos: [3, B, S_pad] + + # THD side: pack with a non-trivial divisor so the padded and + # unpadded cu_seqlens diverge — this exercises the distinction. + for divisible_by in (1, 4): + packed_input_ids, psp, thd_grid, seqlens, seqlens_padded = ( + _pack_samples(samples, divisible_by=divisible_by) + ) + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=thd_grid, + packed_seq_params=psp, + ) + # thd_pos: [3, 1, T] + assert thd_pos.shape == ( + 3, 1, packed_input_ids.shape[1], + ), f"bad THD shape {thd_pos.shape}" + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for k, (valid_len, seg_start) in enumerate( + zip(seqlens, seg_starts) + ): + thd_slice = thd_pos[:, 0, seg_start:seg_start + valid_len] + bshd_slice = bshd_pos[:, k, :valid_len] + assert torch.equal(thd_slice, bshd_slice), ( + f"[divisible_by={divisible_by}] segment {k} " + f"(valid_len={valid_len}, seg_start={seg_start}) " + f"disagrees:\nTHD:\n{thd_slice}\nBSHD:\n{bshd_slice}" + ) + + +# ----------------------------------------------------------------------------- +# Test 3: THD with no images (text-only packed) +# ----------------------------------------------------------------------------- + +def test_thd_text_only_restarts_per_segment(): + """Text-only THD: each segment gets a fresh ``[0..valid_len-1]`` range.""" + samples = [ + _build_sample(prefix_text_len=6, grids=[], suffix_text_len=0), + _build_sample(prefix_text_len=11, grids=[], suffix_text_len=0), + _build_sample(prefix_text_len=3, grids=[], suffix_text_len=0), + ] + packed_input_ids, psp, _, seqlens, seqlens_padded = _pack_samples( + samples, divisible_by=4, + ) + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=None, + packed_seq_params=psp, + ) + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for valid_len, seg_start in zip(seqlens, seg_starts): + expected = ( + torch.arange(valid_len, dtype=thd_pos.dtype) + .view(1, -1) + .expand(3, -1) + ) + got = thd_pos[:, 0, seg_start:seg_start + valid_len] + assert torch.equal(got, expected), ( + f"text-only segment mismatch at seg_start={seg_start}, " + f"valid_len={valid_len}:\n{got}\nexpected:\n{expected}" + ) + + +# ----------------------------------------------------------------------------- +# Test 4: Explicit two-sequence batch with vision, both in BSHD and THD +# ----------------------------------------------------------------------------- + +def _two_image_samples(): + """Two samples, each with one image — the smallest case that can + expose a bug where segment k > 0 positions leak state from segment + k - 1 (e.g. non-restarted ``st_idx`` or a stale ``image_index``). + """ + return [ + _build_sample( + prefix_text_len=5, + grids=[(1, 4, 4)], # 4 image tokens after spatial merge + suffix_text_len=3, + ), + _build_sample( + prefix_text_len=4, + grids=[(1, 4, 4)], + suffix_text_len=6, + ), + ] + + +def test_bshd_batch_size_2_with_vision(): + """BSHD with ``B == 2``: both rows' positions restart at 0 and match + the pinned reference. + """ + samples = _two_image_samples() + max_len = max(s.numel() for s, _ in samples) + + input_ids_rows, mask_rows, grid_rows = [], [], [] + for tokens, grids in samples: + L = tokens.numel() + input_ids_rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + mask_rows.append(m) + grid_rows.append(grids) + + input_ids = torch.stack(input_ids_rows) # [2, S] + attention_mask = torch.stack(mask_rows) + image_grid_thw = torch.cat(grid_rows, dim=0) + assert input_ids.shape[0] == 2 + + old_pos, old_delta = _old_get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + new_pos, new_delta = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + assert torch.equal(old_pos, new_pos), ( + f"BSHD B=2 position mismatch vs reference.\n" + f"old:\n{old_pos}\nnew:\n{new_pos}" + ) + assert torch.equal(old_delta, new_delta) + + # Both rows must start at position 0. + for i in range(2): + valid_len = int(attention_mask[i].sum().item()) + assert torch.all(new_pos[:, i, 0] == 0), ( + f"row {i} does not start at 0: {new_pos[:, i, 0]}" + ) + # Sanity: positions within the valid region are strictly < valid_len + # would be wrong (MRoPE can skip positions), so just check max. + assert new_pos[:, i, :valid_len].max() < valid_len + + +def test_thd_batch_size_2_with_vision(): + """THD with 2 packed sequences: seg 1 positions restart at 0 and + equal BSHD row 1 on the valid region (bit-identical). + """ + samples = _two_image_samples() + + # BSHD reference. + max_len = max(s.numel() for s, _ in samples) + rows, masks, grids_bshd = [], [], [] + for tokens, grids in samples: + L = tokens.numel() + rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + masks.append(m) + grids_bshd.append(grids) + bshd_input_ids = torch.stack(rows) + bshd_mask = torch.stack(masks) + bshd_grid = torch.cat(grids_bshd, dim=0) + bshd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=bshd_input_ids, + image_grid_thw=bshd_grid, + attention_mask=bshd_mask, + ) + + # THD packed version with a non-trivial divisor so padded and + # unpadded cu_seqlens disagree. + packed_input_ids, psp, thd_grid, seqlens, seqlens_padded = ( + _pack_samples(samples, divisible_by=4) + ) + assert len(seqlens) == 2 + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=thd_grid, + packed_seq_params=psp, + ) + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for k, (valid_len, seg_start) in enumerate( + zip(seqlens, seg_starts) + ): + thd_slice = thd_pos[:, 0, seg_start:seg_start + valid_len] + bshd_slice = bshd_pos[:, k, :valid_len] + assert torch.equal(thd_slice, bshd_slice), ( + f"seg {k} THD vs BSHD row {k} mismatch.\n" + f"THD:\n{thd_slice}\nBSHD:\n{bshd_slice}" + ) + # Critical: seg k must start at position 0 (bug 2 check). + assert torch.all(thd_slice[:, 0] == 0), ( + f"seg {k} does not start at 0 — positions leaked from " + f"previous segment: first col = {thd_slice[:, 0]}" + ) + + +if __name__ == "__main__": + test_bshd_matches_old_reference() + print("[ok] test_bshd_matches_old_reference") + test_thd_matches_bshd_padded() + print("[ok] test_thd_matches_bshd_padded") + test_thd_text_only_restarts_per_segment() + print("[ok] test_thd_text_only_restarts_per_segment") + test_bshd_batch_size_2_with_vision() + print("[ok] test_bshd_batch_size_2_with_vision") + test_thd_batch_size_2_with_vision() + print("[ok] test_thd_batch_size_2_with_vision") + print("All parity tests passed.") diff --git a/examples/multimodal_dev/tests/test_thd_correctness.py b/examples/multimodal_dev/tests/test_thd_correctness.py new file mode 100644 index 00000000000..1f92cc2e2f7 --- /dev/null +++ b/examples/multimodal_dev/tests/test_thd_correctness.py @@ -0,0 +1,380 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# This is a stdout-reporting standalone script; `print` is intentional. +# pylint: disable=bad-builtin + +"""BSHD vs THD correctness test for multimodal_dev packed sequence support. + +Validates that packing a [B, S] batch into [1, T] THD format produces +numerically equivalent loss values and gradient norms through a GPTModel. + +The test uses equal-length sequences (no padding) so that BSHD causal +attention and THD cu_seqlens-based causal attention are mathematically +identical. This makes any numerical deviation a real bug rather than an +expected consequence of different padding/masking strategies. + +Usage:: + + # Single GPU (flash attention): + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_thd_correctness.py + + # Override model size: + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_thd_correctness.py \\ + --num-layers 4 --hidden-size 512 --num-heads 8 --num-kv-heads 4 +""" + +import argparse +import os +import sys + +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import pack_or_pad_batch +from examples.multimodal_dev.tests._helpers import grad_norm, mean_loss +from megatron.core.models.gpt import GPTModel +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.transformer_config import TransformerConfig +from tests.unit_tests.test_utilities import Utils + + +def _samples_from_bshd(input_ids, labels, loss_mask, seq_lengths=None): + """Slice ``[B, S]`` tensors into the per-sample dict list that + ``pack_or_pad_batch`` expects. ``seq_lengths`` lets variable-length + samples carry only their valid tokens (no attention_mask needed). + """ + B, S = input_ids.shape + if seq_lengths is None: + seq_lengths = [S] * B + samples = [] + for i, L in enumerate(seq_lengths): + samples.append( + { + "input_ids": input_ids[i, :L].clone(), + "labels": labels[i, :L].clone(), + "loss_mask": loss_mask[i, :L].clone(), + # pack_or_pad_batch requires these keys but the model call + # below ignores them; provide minimal dummies. + "pixel_values": torch.zeros(1, 1, device=input_ids.device), + "image_grid_thw": torch.tensor( + [[2, 1, 1]], dtype=torch.long, device=input_ids.device + ), + } + ) + return samples + + +def _thd_position_ids(seq_lengths, device): + """Build ``[1, T]`` THD position_ids: each segment restarts at 0.""" + return ( + torch.cat([torch.arange(L, device=device) for L in seq_lengths]).unsqueeze(0).contiguous() + ) + + +# =================================================================== +# Helpers +# =================================================================== + + +def _build_model(cfg, vocab_size, max_seq_len): + """Build a small GPTModel for testing.""" + spec = get_gpt_layer_with_transformer_engine_spec() + model = GPTModel( + config=cfg, + transformer_layer_spec=spec, + vocab_size=vocab_size, + max_sequence_length=max_seq_len, + pre_process=True, + post_process=True, + parallel_output=False, + position_embedding_type="rope", + ) + model.cuda() + return model + + +# =================================================================== +# Core test logic +# =================================================================== + + +def run_equal_length_test(model, batch_size, seq_len, vocab_size, seed, atol_loss, rtol_grad): + """Compare BSHD and THD with equal-length sequences (no padding). + + Returns a dict with test metrics for logging. + """ + B, S = batch_size, seq_len + + # Deterministic data generation. + torch.manual_seed(seed + 1) + input_ids = torch.randint(0, vocab_size, (B, S), device="cuda") + labels = torch.randint(0, vocab_size, (B, S), device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + position_ids = torch.arange(S, device="cuda").unsqueeze(0).expand(B, -1).contiguous() + + # ---- BSHD forward / backward ---- + output_bshd = model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + labels=labels, + loss_mask=loss_mask, + ) + bshd_loss = mean_loss(output_bshd, loss_mask) + bshd_loss.backward() + bshd_gn = grad_norm(model) + bshd_lv = bshd_loss.item() + bshd_per_token = output_bshd.detach().float().view(-1).clone() + + model.zero_grad() + + # ---- THD forward / backward ---- + samples = _samples_from_bshd(input_ids, labels, loss_mask) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + thd_position_ids = _thd_position_ids([S] * B, device="cuda") + + output_thd = model( + input_ids=packed["input_ids"], + position_ids=thd_position_ids, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + packed_seq_params=psp, + ) + thd_loss = mean_loss(output_thd, packed["loss_mask"]) + thd_loss.backward() + thd_gn = grad_norm(model) + thd_lv = thd_loss.item() + thd_per_token = output_thd.detach().float().view(-1).clone() + + model.zero_grad() + + # ---- Comparison ---- + loss_diff = abs(bshd_lv - thd_lv) + grad_diff = abs(bshd_gn - thd_gn) + grad_rel = grad_diff / max(bshd_gn, 1e-8) + token_max_diff = (bshd_per_token - thd_per_token).abs().max().item() + token_mean_diff = (bshd_per_token - thd_per_token).abs().mean().item() + + loss_ok = loss_diff < atol_loss + grad_ok = grad_rel < rtol_grad + + metrics = dict( + bshd_loss=bshd_lv, + thd_loss=thd_lv, + loss_diff=loss_diff, + bshd_grad_norm=bshd_gn, + thd_grad_norm=thd_gn, + grad_diff=grad_diff, + grad_rel=grad_rel, + token_max_diff=token_max_diff, + token_mean_diff=token_mean_diff, + loss_ok=loss_ok, + grad_ok=grad_ok, + ) + return metrics + + +def run_variable_length_smoke_test(model, vocab_size, seed): + """Smoke test: variable-length sequences packed to THD. + + Does NOT compare against BSHD (padding in BSHD changes attention + context). Validates that: + - Packing produces correct shapes + - Forward + backward complete without error + - Loss is finite + - Gradients are finite and non-zero + + Returns a dict with test metrics. + """ + seq_lengths = [128, 96, 112, 80] + S = max(seq_lengths) + B = len(seq_lengths) + + torch.manual_seed(seed + 2) + input_ids = torch.randint(0, vocab_size, (B, S), device="cuda") + labels = torch.randint(0, vocab_size, (B, S), device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + + # Mask out padded positions in loss_mask for the input we hand to + # pack_or_pad_batch (variable-length samples carry only valid tokens). + for i, sl in enumerate(seq_lengths): + loss_mask[i, sl:] = 0.0 + + samples = _samples_from_bshd(input_ids, labels, loss_mask, seq_lengths=seq_lengths) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + thd_position_ids = _thd_position_ids(seq_lengths, device="cuda") + + T = sum(seq_lengths) + assert packed["input_ids"].shape == ( + 1, + T, + ), f"Expected [1, {T}], got {packed['input_ids'].shape}" + assert packed["labels"].shape == (1, T) + assert packed["loss_mask"].shape == (1, T) + assert psp.cu_seqlens_q.tolist() == [ + 0, + seq_lengths[0], + seq_lengths[0] + seq_lengths[1], + seq_lengths[0] + seq_lengths[1] + seq_lengths[2], + T, + ] + + output = model( + input_ids=packed["input_ids"], + position_ids=thd_position_ids, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + packed_seq_params=psp, + ) + loss = mean_loss(output, packed["loss_mask"]) + loss.backward() + gn = grad_norm(model) + loss_val = loss.item() + + model.zero_grad() + + loss_finite = torch.isfinite(torch.tensor(loss_val)).item() + grad_finite = torch.isfinite(torch.tensor(gn)).item() + grad_nonzero = gn > 0 + + return dict( + loss=loss_val, + grad_norm=gn, + total_tokens=T, + loss_finite=loss_finite, + grad_finite=grad_finite, + grad_nonzero=grad_nonzero, + passed=loss_finite and grad_finite and grad_nonzero, + ) + + +# =================================================================== +# Main +# =================================================================== + + +def _print_banner(title): + print(f"\n{'='*60}") + print(f" {title}") + print(f"{'='*60}") + + +def main(): + """CLI entry: run the equal-length parity test + variable-length smoke test.""" + parser = argparse.ArgumentParser(description="BSHD vs THD correctness test") + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seq-len", type=int, default=128) + parser.add_argument("--vocab-size", type=int, default=1024) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--num-layers", type=int, default=2) + parser.add_argument("--num-heads", type=int, default=4) + parser.add_argument("--num-kv-heads", type=int, default=2) + parser.add_argument("--ffn-hidden-size", type=int, default=512) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--atol-loss", type=float, default=1e-5, help="Absolute tolerance for loss comparison" + ) + parser.add_argument( + "--rtol-grad", type=float, default=1e-3, help="Relative tolerance for grad norm comparison" + ) + args = parser.parse_args() + + Utils.initialize_model_parallel(tensor_model_parallel_size=1) + model_parallel_cuda_manual_seed(args.seed) + + config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + ffn_hidden_size=args.ffn_hidden_size, + num_attention_heads=args.num_heads, + num_query_groups=args.num_kv_heads, + bf16=True, + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + hidden_dropout=0.0, + attention_dropout=0.0, + tensor_model_parallel_size=1, + sequence_parallel=False, + ) + + model = _build_model(config, args.vocab_size, args.seq_len) + + all_passed = True + + # ---------------------------------------------------------------- + # Test 1: equal-length correctness (BSHD vs THD) + # ---------------------------------------------------------------- + _print_banner("Test 1: Equal-length BSHD vs THD correctness") + m = run_equal_length_test( + model=model, + batch_size=args.batch_size, + seq_len=args.seq_len, + vocab_size=args.vocab_size, + seed=args.seed, + atol_loss=args.atol_loss, + rtol_grad=args.rtol_grad, + ) + print( + f" Config: B={args.batch_size}, S={args.seq_len}, " + f"H={args.hidden_size}, L={args.num_layers}, " + f"heads={args.num_heads}/{args.num_kv_heads}" + ) + print(f" BSHD loss: {m['bshd_loss']:.8f}") + print(f" THD loss: {m['thd_loss']:.8f}") + print(f" Loss abs diff: {m['loss_diff']:.2e}") + print(f" BSHD grad norm: {m['bshd_grad_norm']:.8f}") + print(f" THD grad norm: {m['thd_grad_norm']:.8f}") + print(f" Grad norm rel diff: {m['grad_rel']:.2e}") + print(f" Per-token max diff: {m['token_max_diff']:.2e}") + print(f" Per-token mean diff: {m['token_mean_diff']:.2e}") + print( + f" Loss match: {'PASS' if m['loss_ok'] else 'FAIL'} " f"(atol={args.atol_loss})" + ) + print( + f" Grad norm match: {'PASS' if m['grad_ok'] else 'FAIL'} " f"(rtol={args.rtol_grad})" + ) + if not (m["loss_ok"] and m["grad_ok"]): + all_passed = False + + # ---------------------------------------------------------------- + # Test 2: variable-length smoke test (THD only) + # ---------------------------------------------------------------- + _print_banner("Test 2: Variable-length THD smoke test") + v = run_variable_length_smoke_test(model, args.vocab_size, args.seed) + print(f" Seq lengths: [128, 96, 112, 80]") + print(f" Total packed tokens: {v['total_tokens']}") + print(f" Loss: {v['loss']:.8f}") + print(f" Grad norm: {v['grad_norm']:.8f}") + print(f" Loss finite: {'PASS' if v['loss_finite'] else 'FAIL'}") + print(f" Grad finite: {'PASS' if v['grad_finite'] else 'FAIL'}") + print(f" Grad nonzero: {'PASS' if v['grad_nonzero'] else 'FAIL'}") + if not v["passed"]: + all_passed = False + + # ---------------------------------------------------------------- + # Summary + # ---------------------------------------------------------------- + _print_banner("Summary") + if all_passed: + print(" ALL TESTS PASSED") + else: + print(" SOME TESTS FAILED") + print(f"{'='*60}\n") + + Utils.destroy_model_parallel() + + if not all_passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_thd_e2e.py b/examples/multimodal_dev/tests/test_thd_e2e.py new file mode 100644 index 00000000000..c8d9f2e1622 --- /dev/null +++ b/examples/multimodal_dev/tests/test_thd_e2e.py @@ -0,0 +1,314 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Tests for THD / padded batch construction in multimodal_dev. + +Exercises the production data path :func:`pack_or_pad_batch`, which +consumes a list of per-sample dicts produced by the dataset's +``__getitem__`` and produces either a packed THD batch (``[1, T]``) or a +padded BSHD batch (``[B, S]``). + +``pack_or_pad_batch`` ends with a TP-group broadcast, so these tests +require ``torch.distributed`` to be initialised. Run via:: + + torchrun --nproc-per-node 1 -m pytest -q \\ + examples/multimodal_dev/tests/test_thd_e2e.py +""" + +import os +import sys + +import pytest +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import _build_packed_seq_params, pack_or_pad_batch +from tests.unit_tests.test_utilities import Utils + + +@pytest.fixture(scope="module", autouse=True) +def _init_model_parallel(): + """Single-rank TP init so pack_or_pad_batch's TP broadcast is a no-op.""" + Utils.initialize_model_parallel(tensor_model_parallel_size=1) + yield + Utils.destroy_model_parallel() + + +def _make_sample( + seq_len: int, *, base: int = 0, num_patches: int = 4, pixel_dim: int = 8, device: str = "cuda" +): + """Per-sample dict in the shape produced by ``CordV2VLMDataset.__getitem__``. + + 1-D ``input_ids`` / ``labels`` / ``loss_mask`` over the sequence dim; + ``pixel_values`` is ``[num_patches, pixel_dim]``; ``image_grid_thw`` is + ``[1, 3]``. + """ + return { + "input_ids": torch.arange(seq_len, dtype=torch.long, device=device) + base, + "labels": (torch.arange(seq_len, dtype=torch.long, device=device) + base + 100), + "loss_mask": torch.ones(seq_len, dtype=torch.float, device=device), + "pixel_values": torch.full((num_patches, pixel_dim), float(base), device=device), + "image_grid_thw": torch.tensor([[2, 4, 4]], dtype=torch.long, device=device), + } + + +# =================================================================== +# _build_packed_seq_params — pure helper, exercised independently +# =================================================================== + + +class TestBuildPackedSeqParams: + """Tests for ``_build_packed_seq_params``.""" + + def test_basic(self): + """Mixed-length sample build sanity check.""" + params = _build_packed_seq_params(torch.tensor([5, 3, 7], dtype=torch.int32), device="cpu") + assert params.qkv_format == "thd" + assert params.cu_seqlens_q.tolist() == [0, 5, 8, 15] + assert params.cu_seqlens_kv.tolist() == [0, 5, 8, 15] + assert params.max_seqlen_q == 7 + assert params.max_seqlen_kv == 7 + assert params.total_tokens == 15 + assert params.cu_seqlens_q_padded.tolist() == [0, 5, 8, 15] + assert params.cu_seqlens_kv_padded.tolist() == [0, 5, 8, 15] + + def test_equal_lengths(self): + """Equal-length samples produce uniform cu_seqlens.""" + params = _build_packed_seq_params(torch.tensor([4, 4, 4], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.tolist() == [0, 4, 8, 12] + assert params.max_seqlen_q == 4 + assert params.total_tokens == 12 + + def test_single_sample(self): + """Single-sample batch round-trips its own length.""" + params = _build_packed_seq_params(torch.tensor([10], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.tolist() == [0, 10] + assert params.max_seqlen_q == 10 + assert params.total_tokens == 10 + + def test_dtype_is_int32(self): + """``cu_seqlens_q`` is cast to int32 regardless of input dtype.""" + params = _build_packed_seq_params(torch.tensor([3, 5], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.dtype == torch.int32 + + def test_seq_idx_computed(self): + """``__post_init__`` computes ``seq_idx`` for Mamba compatibility.""" + params = _build_packed_seq_params(torch.tensor([3, 2], dtype=torch.int32), device="cpu") + assert params.seq_idx is not None + assert params.seq_idx.shape == (1, 5) + assert params.seq_idx[0].tolist() == [0, 0, 0, 1, 1] + + +# =================================================================== +# pack_or_pad_batch — packed (THD) mode +# =================================================================== + + +class TestPackOrPadBatchPacked: + """``pack_or_pad_batch(..., use_packed_sequence=True)`` produces ``[1, T]``.""" + + def test_equal_lengths(self): + """Two equal-length samples → packed ``[1, 2S]``.""" + S = 8 + batch = [_make_sample(S, base=0), _make_sample(S, base=1000)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = 2 * S + assert packed["input_ids"].shape == (1, T) + assert packed["labels"].shape == (1, T) + assert packed["loss_mask"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, S, T] + assert psp.cu_seqlens_q_padded.tolist() == [0, S, T] + assert psp.max_seqlen_q == S + assert psp.total_tokens == T + + def test_variable_lengths(self): + """Variable-length samples concatenated end-to-end.""" + lens = [5, 8, 3] + batch = [_make_sample(L, base=i * 1000) for i, L in enumerate(lens)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = sum(lens) + assert packed["input_ids"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 5, 13, 16] + assert psp.cu_seqlens_q_padded.tolist() == [0, 5, 13, 16] + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == T + + def test_token_order_preserved(self): + """Sample 0's tokens precede sample 1's tokens in the packed sequence.""" + s0 = _make_sample(3, base=10) + s1 = _make_sample(3, base=40) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["input_ids"][0].tolist() == [10, 11, 12, 40, 41, 42] + + def test_labels_loss_mask_content_preserved(self): + """labels and loss_mask carry through unchanged when divisible_by=1.""" + s = _make_sample(4, base=0) + packed = pack_or_pad_batch([s], use_packed_sequence=True, device="cuda") + assert packed["labels"][0].tolist() == [100, 101, 102, 103] + assert packed["loss_mask"][0].tolist() == [1.0, 1.0, 1.0, 1.0] + + def test_pixel_values_concatenated(self): + """``pixel_values`` are concatenated along the patch dim.""" + s0 = _make_sample(4, base=0, num_patches=4, pixel_dim=8) + s1 = _make_sample(4, base=10, num_patches=6, pixel_dim=8) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["pixel_values"].shape == (10, 8) + assert packed["pixel_values"][:4].eq(0.0).all().item() + assert packed["pixel_values"][4:].eq(10.0).all().item() + + def test_image_grid_thw_concatenated(self): + """``image_grid_thw`` rows are concatenated along the first dim.""" + s0 = _make_sample(4, base=0) + s1 = _make_sample(4, base=10) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["image_grid_thw"].shape == (2, 3) + + def test_single_sample_round_trip(self): + """A single sample packs to its own length.""" + s = _make_sample(7, base=0) + packed = pack_or_pad_batch([s], use_packed_sequence=True, device="cuda") + assert packed["input_ids"].shape == (1, 7) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 7] + assert psp.max_seqlen_q == 7 + assert psp.total_tokens == 7 + + +# =================================================================== +# pack_or_pad_batch — padded (BSHD) mode +# =================================================================== + + +class TestPackOrPadBatchPadded: + """``pack_or_pad_batch(..., use_packed_sequence=False)`` produces ``[B, S]``.""" + + def test_equal_lengths(self): + """Equal-length samples → ``[B, S]`` without further padding.""" + S = 6 + batch = [_make_sample(S, base=0), _make_sample(S, base=10)] + padded = pack_or_pad_batch(batch, use_packed_sequence=False, seq_length=S, device="cuda") + assert padded["input_ids"].shape == (2, S) + assert padded["labels"].shape == (2, S) + assert padded["loss_mask"].shape == (2, S) + # Sample-0 content is preserved verbatim. + assert padded["input_ids"][0].tolist() == list(range(S)) + + def test_pads_short_sample_to_batch_max(self): + """Shorter sample is right-padded to match the batch max length.""" + long_sample = _make_sample(7, base=0) + short_sample = _make_sample(3, base=10) + padded = pack_or_pad_batch( + [long_sample, short_sample], use_packed_sequence=False, seq_length=7, device="cuda" + ) + assert padded["input_ids"].shape == (2, 7) + # Short sample: original [10, 11, 12] then pad zeros. + assert padded["input_ids"][1].tolist() == [10, 11, 12, 0, 0, 0, 0] + # labels pad with -100 (ignore index). + assert padded["labels"][1].tolist() == [110, 111, 112, -100, -100, -100, -100] + # loss_mask pads with 0. + assert padded["loss_mask"][1].tolist() == [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0] + + def test_seq_length_required(self): + """``seq_length`` must be provided in padded mode.""" + s = _make_sample(4, base=0) + with pytest.raises(AssertionError, match="seq_length"): + pack_or_pad_batch([s], use_packed_sequence=False, seq_length=None, device="cuda") + + def test_pixel_values_concatenated(self): + """``pixel_values`` concat preserves both samples' patches.""" + s0 = _make_sample(4, base=0, num_patches=4, pixel_dim=8) + s1 = _make_sample(4, base=10, num_patches=6, pixel_dim=8) + padded = pack_or_pad_batch([s0, s1], use_packed_sequence=False, seq_length=4, device="cuda") + assert padded["pixel_values"].shape == (10, 8) + assert padded["pixel_values"][:4].eq(0.0).all().item() + assert padded["pixel_values"][4:].eq(10.0).all().item() + + +# =================================================================== +# pack_or_pad_batch — divisible_by = 4 alignment +# =================================================================== + + +class TestPackOrPadBatchDivisibleBy4: + """Per-sample sequence alignment when ``divisible_by = 4``. + + The function computes ``divisible_by`` from the parallel state. With + ``world_size=1`` we cannot stand up a real CP=2 group, so we patch + :func:`mpu.get_context_parallel_world_size` to return 2; the function + then takes the ``cp_size > 1`` branch and yields + ``divisible_by = cp_size * 2 = 4`` (no SP). + """ + + @pytest.fixture + def cp2(self, monkeypatch): + """Patch CP world size to 2 so ``divisible_by = 4``.""" + from examples.multimodal_dev import forward_step + + monkeypatch.setattr(forward_step.mpu, "get_context_parallel_world_size", lambda: 2) + + def test_packed_aligned_samples_no_padding(self, cp2): + """Samples already multiples of 4 → cu_seqlens == cu_seqlens_padded.""" + batch = [_make_sample(8, base=0), _make_sample(4, base=100)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = 12 + assert packed["input_ids"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 8, 12] + assert psp.cu_seqlens_q_padded.tolist() == [0, 8, 12] + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == 12 + + def test_packed_misaligned_samples_padded_per_sample(self, cp2): + """Each sample padded up to the nearest multiple of 4.""" + # lens=[5, 8, 3] → padded=[8, 8, 4] → T_padded = 20. + batch = [_make_sample(5, base=0), _make_sample(8, base=100), _make_sample(3, base=200)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T_padded = 20 + assert packed["input_ids"].shape == (1, T_padded) + psp = packed["packed_seq_params"] + # cu_seqlens reflects real per-sample lengths. + assert psp.cu_seqlens_q.tolist() == [0, 5, 13, 16] + # cu_seqlens_padded reflects per-sample alignment to 4. + assert psp.cu_seqlens_q_padded.tolist() == [0, 8, 16, 20] + # max_seqlen comes from the padded lengths. + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == T_padded + + def test_packed_pad_values(self, cp2): + """Pad slots filled with input_ids=0, labels=-100, loss_mask=0.""" + # Single sample len=3 → target_len=4 → 1 pad slot at position 3. + batch = [_make_sample(3, base=10)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + assert packed["input_ids"].shape == (1, 4) + assert packed["input_ids"][0].tolist() == [10, 11, 12, 0] + assert packed["labels"][0].tolist() == [110, 111, 112, -100] + assert packed["loss_mask"][0].tolist() == [1.0, 1.0, 1.0, 0.0] + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 3] + assert psp.cu_seqlens_q_padded.tolist() == [0, 4] + assert psp.max_seqlen_q == 4 + assert psp.total_tokens == 4 + + def test_padded_target_rounded_up_to_multiple_of_4(self, cp2): + """Padded (BSHD) mode: ``target = ceil(min(max, seq_length) / 4) * 4``.""" + # lens=[5, 3], seq_length=10 → min(5, 10) = 5 → ceil(5/4)*4 = 8. + long_sample = _make_sample(5, base=0) + short_sample = _make_sample(3, base=10) + padded = pack_or_pad_batch( + [long_sample, short_sample], use_packed_sequence=False, seq_length=10, device="cuda" + ) + + assert padded["input_ids"].shape == (2, 8) + assert padded["input_ids"][0].tolist() == [0, 1, 2, 3, 4, 0, 0, 0] + assert padded["input_ids"][1].tolist() == [10, 11, 12, 0, 0, 0, 0, 0] + assert padded["labels"][1].tolist() == [110, 111, 112, -100, -100, -100, -100, -100] + assert padded["loss_mask"][1].tolist() == [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] diff --git a/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py b/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py new file mode 100644 index 00000000000..62c908ea91d --- /dev/null +++ b/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Numerical parity for ``Qwen35VLPatchMerger`` vs HuggingFace reference. + +The HuggingFace reference (``Qwen3VLVisionPatchMerger`` in +``transformers/models/qwen3_vl/modeling_qwen3_vl.py``, branch +``use_postshuffle_norm=False``) is reproduced inline so this test does not +require ``transformers`` to be installed. The HF module is verbatim:: + + self.norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.linear_fc1 = nn.Linear(merge_dim, merge_dim) + self.act_fn = nn.GELU() # default approximate='none' + self.linear_fc2 = nn.Linear(merge_dim, out_hidden_size) + + x = self.norm(x) + x = x.view(-1, merge_dim) + x = self.linear_fc2(self.act_fn(self.linear_fc1(x))) + +With matching dims and weights copied across, the Megatron +implementation must agree: + + * fp32 forward: max-abs diff <= 1e-4 (TE LayerNorm vs nn.LayerNorm + have different fused reduction order; ~1e-5 absolute residual is + structural and not a real divergence) + * bf16 forward: max-abs diff <= 5e-2 (bf16 ceiling for two-layer MLP) + +Run with:: + + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_vision_patch_merger_parity.py +""" + +import os +import sys + +import torch +import torch.distributed as dist +import torch.nn as nn + +_REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../.."), +) +if _REPO_ROOT in sys.path: + sys.path.remove(_REPO_ROOT) +sys.path.insert(0, _REPO_ROOT) + +from megatron.core import parallel_state as ps +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.transformer_config import TransformerConfig + +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import Qwen35VLPatchMerger + + +# Match Qwen3.5-VL 9B / 397B-A17B vision tower dims. +HIDDEN_SIZE = 1152 +OUT_HIDDEN_SIZE = 3584 +SPATIAL_MERGE_SIZE = 2 +NUM_PATCHES = 64 # must be divisible by spatial_merge_size ** 2 + +ATOL_FP32 = 1e-4 +RTOL_FP32 = 1e-3 +ATOL_BF16 = 5e-2 +RTOL_BF16 = 5e-2 + + +class HFPatchMergerReference(nn.Module): + """Inline HF ``Qwen3VLVisionPatchMerger`` (use_postshuffle_norm=False).""" + + def __init__(self, hidden_size: int, out_hidden_size: int, spatial_merge_size: int): + super().__init__() + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + self.norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.linear_fc1 = nn.Linear(self.merge_dim, self.merge_dim) + self.act_fn = nn.GELU() # approximate='none' by default + self.linear_fc2 = nn.Linear(self.merge_dim, out_hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + x = self.norm(hidden_states) + x = x.view(-1, self.merge_dim) + x = self.linear_fc1(x) + x = self.act_fn(x) + x = self.linear_fc2(x) + return x + + +def _init_distributed() -> int: + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + return local_rank + + +def _init_megatron_parallel() -> None: + ps.destroy_model_parallel() + ps.initialize_model_parallel(tensor_model_parallel_size=1) + model_parallel_cuda_manual_seed(42) + + +def _build_config(dtype: torch.dtype) -> TransformerConfig: + is_bf16 = dtype is torch.bfloat16 + return TransformerConfig( + num_layers=1, + hidden_size=HIDDEN_SIZE, + ffn_hidden_size=HIDDEN_SIZE, + num_attention_heads=8, + kv_channels=HIDDEN_SIZE // 8, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + sequence_parallel=False, + bf16=is_bf16, + params_dtype=dtype, + pipeline_dtype=dtype, + add_bias_linear=True, + gated_linear_unit=False, + normalization="LayerNorm", + layernorm_epsilon=1e-6, + attention_dropout=0.0, + hidden_dropout=0.0, + ) + + +def _copy_hf_to_megatron(hf: HFPatchMergerReference, mcore: Qwen35VLPatchMerger) -> None: + """TP=1: 1:1 parameter copy between HF nn.Module and the MCore module.""" + with torch.no_grad(): + mcore.patch_norm.weight.copy_(hf.norm.weight.to(mcore.patch_norm.weight.dtype)) + mcore.patch_norm.bias.copy_(hf.norm.bias.to(mcore.patch_norm.bias.dtype)) + mcore.linear_fc1.weight.copy_(hf.linear_fc1.weight.to(mcore.linear_fc1.weight.dtype)) + mcore.linear_fc1.bias.copy_(hf.linear_fc1.bias.to(mcore.linear_fc1.bias.dtype)) + mcore.linear_fc2.weight.copy_(hf.linear_fc2.weight.to(mcore.linear_fc2.weight.dtype)) + mcore.linear_fc2.bias.copy_(hf.linear_fc2.bias.to(mcore.linear_fc2.bias.dtype)) + + +def _run_one(dtype: torch.dtype, atol: float, rtol: float, device: torch.device, seed: int = 42) -> None: + torch.manual_seed(seed) + + hf_ref = HFPatchMergerReference( + hidden_size=HIDDEN_SIZE, + out_hidden_size=OUT_HIDDEN_SIZE, + spatial_merge_size=SPATIAL_MERGE_SIZE, + ).to(device=device, dtype=dtype).eval() + + config = _build_config(dtype) + mcore = Qwen35VLPatchMerger( + config=config, + hidden_size=HIDDEN_SIZE, + out_hidden_size=OUT_HIDDEN_SIZE, + spatial_merge_size=SPATIAL_MERGE_SIZE, + ).to(device=device, dtype=dtype).eval() + + _copy_hf_to_megatron(hf_ref, mcore) + + x = torch.randn(NUM_PATCHES, HIDDEN_SIZE, device=device, dtype=dtype) + + with torch.no_grad(): + y_hf = hf_ref(x) + y_mcore = mcore(x) + + assert y_hf.shape == y_mcore.shape, (y_hf.shape, y_mcore.shape) + diff = (y_hf - y_mcore).abs() + print( + f"[{dtype}] shape={tuple(y_hf.shape)} " + f"max_abs_diff={diff.max().item():.3e} " + f"mean_abs_diff={diff.mean().item():.3e} " + f"hf_norm={y_hf.float().norm().item():.4f} " + f"mcore_norm={y_mcore.float().norm().item():.4f}" + ) + torch.testing.assert_close(y_mcore, y_hf, atol=atol, rtol=rtol) + + +def main() -> None: + local_rank = _init_distributed() + _init_megatron_parallel() + device = torch.device(f"cuda:{local_rank}") + + _run_one(torch.float32, ATOL_FP32, RTOL_FP32, device) + _run_one(torch.bfloat16, ATOL_BF16, RTOL_BF16, device) + + if int(os.environ.get("RANK", 0)) == 0: + print( + "\nPASS: Qwen35VLPatchMerger logits match HF Qwen3VLVisionPatchMerger " + "in both fp32 and bf16." + ) + + +if __name__ == "__main__": + main() diff --git a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py index a852ef226b4..2b02b1a9f57 100644 --- a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py +++ b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py @@ -68,6 +68,7 @@ class FullyShardedDataParallel(_BaseDataParallel): _MODULE_TYPE_REGISTRY: Dict[str, set] = { "column": { "ColumnParallelLinear", + "LinearCrossEntropyModule", "TEColumnParallelLinear", "TELayerNormColumnParallelLinear", "TEColumnParallelGroupedLinear", diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py index 54f5a28ce68..1c71cd33c74 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py @@ -1242,6 +1242,9 @@ def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bo """ self._replace_param_with_raw_if_needed() + if self.data_parallel_sharding_strategy == "no_shard": + return + if not force_sync and self.ddp_config.overlap_param_gather: # All-gather the first bucket before the forward pass. if self.ddp_config.fsdp_all_gather_in_start_param_sync: diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py index 6c907bc25ec..2d7f573afdb 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py @@ -1404,6 +1404,29 @@ def _does_param_require_new_bucket(param): is_expert_parameter = lambda n, p: ".experts." in n + def _should_split_from_grouped_expert_bucket( + is_expert_param: bool, + param: torch.nn.Parameter, + param_chunk_size_factor: int, + chunk_size_factor: int, + same_factor_params: List[torch.nn.Parameter], + ) -> bool: + """ + Split grouped expert (>=3D) tensors with heterogeneous chunk size + factors into separate buckets to avoid LCM-inflated bucket alignment + padding. + """ + # Non-expert groups keep the original LCM/fragment merge. + if not is_expert_param: + return False + # Param already aligns with bucket chunk size factor (always true for + # the first param after sort); no split needed. + if param_chunk_size_factor == chunk_size_factor: + return False + return to_local_if_dtensor(param).dim() >= 3 or any( + to_local_if_dtensor(p).dim() >= 3 for p in same_factor_params + ) + # Step 1: Group the parameters according to their execution order and attributes. # FSDP unit module parameters are split into multiple parameter sub-groups. # All parameters in the module are assigned a parameter group, even non-FSDP modules. @@ -1503,17 +1526,27 @@ def _does_param_require_new_bucket(param): remaining_params = [] for param in params: param_shape = to_local_if_dtensor(param).shape + param_chunk_size_factor = param_shape[1:].numel() + if _should_split_from_grouped_expert_bucket( + group.is_expert_param, + param, + param_chunk_size_factor, + chunk_size_factor, + same_factor_params, + ): + remaining_params.append(param) + continue if ( - param_shape[1:].numel() == chunk_size_factor + param_chunk_size_factor == chunk_size_factor or ( - chunk_size_factor % param_shape[1:].numel() == 0 + chunk_size_factor % param_chunk_size_factor == 0 and param_shape.numel() % chunk_size_factor == 0 ) or (param_shape.numel() < chunk_size_factor) ): same_factor_params.append(param) else: - lcm_chunk_size_factor = math.lcm(chunk_size_factor, param_shape[1:].numel()) + lcm_chunk_size_factor = math.lcm(chunk_size_factor, param_chunk_size_factor) chunk_size_factor = lcm_chunk_size_factor same_factor_params.append(param) # Create a new parameter group with the same chunk size factor. @@ -3252,7 +3285,7 @@ def all_reduce_gradients(self, async_op: bool = False): all_reduce_ops = [] for g in self.parameter_groups: gbuf = g.main_grad_buffer - if gbuf is not None: + if gbuf is None: continue scaling_factor = gbuf.gradient_scaling_factor if self.ddp_config.check_for_nan_in_grad: diff --git a/megatron/core/fusions/fused_mega_pre_gated_delta_rule.py b/megatron/core/fusions/fused_mega_pre_gated_delta_rule.py new file mode 100644 index 00000000000..c97036fca12 --- /dev/null +++ b/megatron/core/fusions/fused_mega_pre_gated_delta_rule.py @@ -0,0 +1,1104 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Mega fused pre-gated-delta-rule kernels. + +This is the "mega" sibling of :mod:`fused_pre_gated_delta_rule`. The streamed +path splits the pre-gated-delta-rule front-end into four separate Triton launch +scopes (QK / V / Z / g-beta) plus an external conv backward, optimized for +per-kernel quality and overlap under CUDA-graph capture. The mega path instead +folds **all forward tasks into a single Triton launch** using a flat logical +task space, trading a little per-kernel efficiency for far fewer host-side +launches. It is the right choice for non-CUDA-graph recipes where launch +overhead is visible in the trace. + +Public contract is identical across unfused / streamed / mega: +``(query, key, value, gate, beta, g)``. + +Design notes: + +* The forward kernel maps ``program_id(0)`` onto a flat row space partitioned + into QK, V, Z, and g/beta ranges; ``program_id(1)`` tiles the sequence axis. + Each program inspects its row id and runs exactly one task body. This keeps + every sub-computation in one launch while still letting Triton schedule + memory-bound (Z copy) and compute-bound (QK/V conv) tiles concurrently on the + SMs. +* Numerics mirror the streamed/unfused reference **bit-for-bit within the unit + test tolerance**: the conv accumulator is rounded through the activation + dtype before SiLU; the SiLU output is rounded again before the L2-norm + reduction; ``g`` uses an fp32 ``log(1+exp(...))`` softplus; ``beta`` uses an + fp32 sigmoid. The QK ``silu(conv(x))`` intermediate is persisted channel-last + exactly as the streamed path saves it, so the backward can be shared. +* The kernel assumes ``key_head_dim == value_head_dim`` so a single + ``HEAD_DIM`` constexpr drives the QK/V/Z channel tiles (true for the GDN + production shapes and the unit tests). This is asserted at the Python entry. + +The backward mirrors the forward: a single fused Triton kernel folds the four +streamed branch backward scopes (QK l2norm/repeat, V layout, Z layout, g/beta +chain rule) into one flat-task launch, then the depthwise conv input/weight +gradients are delegated to the same external ``causal_conv1d_bwd_function`` the +streamed path uses (its hand-tuned C++ remains the conv-backward anchor). That +is two launches total (one fused branch kernel + one external conv backward), +down from the streamed path's five, while staying numerically bit-identical to +the streamed branch kernels. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl +from torch import Tensor + +# Reuse the streamed module's validated constants and the external conv backward +# binding. Importing is not a modification of that module. +from megatron.core.fusions.fused_pre_gated_delta_rule import ( + _L2NORM_EPS, + _causal_conv1d_bwd_function, + _is_power_of_two, + _resolve_packed_seq_idx, +) + + +# --------------------------------------------------------------------------- +# Forward kernel +# --------------------------------------------------------------------------- + + +def _mega_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ] + + +@triton.jit +def _mega_seq_bounds(cu_seqlens_ptr, token_offsets, total_tokens, num_packed_seqs): + """Lane-wise packed-sequence [start, end) bounds for flattened THD tokens. + + Local copy of the streamed helper so this kernel never depends on + cross-module ``@triton.jit`` symbol resolution. + """ + + safe_tokens = tl.minimum(token_offsets, total_tokens - 1) + seq_start = token_offsets * 0 + seq_end = token_offsets * 0 + total_tokens + + seq_id = 0 + while seq_id < num_packed_seqs: + start = tl.load(cu_seqlens_ptr + seq_id) + end = tl.load(cu_seqlens_ptr + seq_id + 1) + in_seq = (safe_tokens >= start) & (safe_tokens < end) + seq_start = tl.where(in_seq, start, seq_start) + seq_end = tl.where(in_seq, end, seq_end) + seq_id += 1 + + return seq_start, seq_end + + +@triton.autotune( + configs=_mega_autotune_configs(), + key=["seq_len", "HEAD_DIM", "K_W", "num_key_heads", "num_value_heads", "REPEAT", "HAS_THD"], +) +@triton.jit +def _mega_forward_kernel( + qkvzba_ptr, + weight_ptr, + A_log_ptr, + dt_bias_ptr, + qk_out_ptr, + value_ptr, + gate_ptr, + g_ptr, + beta_ptr, + silu_save_ptr, + cu_seqlens_ptr, + seq_len, + num_packed_seqs, + num_key_heads, + num_value_heads, + qk_channels, + v_channels, + R_qk, + R_v, + R_z, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + qk_g_stride, + qk_b_stride, + qk_s_stride, + qk_h_stride, + v_b_stride, + v_s_stride, + v_h_stride, + z_b_stride, + z_s_stride, + z_h_stride, + g_b_stride, + g_s_stride, + g_h_stride, + beta_b_stride, + beta_s_stride, + beta_h_stride, + silu_b_stride, + silu_c_stride, + silu_s_stride, + eps, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + HAS_THD: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """All-in-one forward for the pre-gated-delta-rule front-end. + + Flat task space on ``program_id(0)``: + rows [0, R_qk) -> QK conv+silu+l2norm+repeat + rows [R_qk, R_qk+R_v) -> V conv+silu + rows [R_qk+R_v, +R_z) -> Z copy + rows [.., end) -> g/beta + ``program_id(1)`` tiles the (flattened, for THD) sequence axis. + """ + + pid_row = tl.program_id(0) + pid_s = tl.program_id(1) + + # Common rounding dtype (activation dtype, e.g. bf16). All q/k/v/gate/beta + # outputs share this; g is fp32. + out_ty = qk_out_ptr.dtype.element_ty + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + chan_off = tl.arange(0, HEAD_DIM) + + R_qkv = R_qk + R_v + R_qkvz = R_qkv + R_z + + if pid_row < R_qk: + # ---- QK: depthwise causal conv + silu + l2norm + head repeat ---- + local = pid_row + heads_per_batch = 2 * num_key_heads + batch_id = local // heads_per_batch + lb = local - batch_id * heads_per_batch + group_id = lb // num_key_heads # 0 -> Q, 1 -> K + head_id = lb - group_id * num_key_heads + chan = group_id * qk_channels + head_id * HEAD_DIM + chan_off + + if HAS_THD: + seq_start, seq_end = _mega_seq_bounds( + cu_seqlens_ptr, s_offs, seq_len, num_packed_seqs + ) + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + if HAS_THD: + x_mask = s_mask & (x_s >= seq_start) & (x_s < seq_end) + safe_x_s = tl.minimum(tl.maximum(x_s, 0), seq_len - 1) + else: + x_mask = (x_s >= 0) & (x_s < seq_len) + safe_x_s = x_s + x_ptr = ( + qkvzba_ptr + + safe_x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc = acc.to(out_ty).to(tl.float32) # F.conv1d rounding + silu_out = acc * tl.sigmoid(acc) + silu_out = silu_out.to(out_ty).to(tl.float32) # round before l2norm + + # Persist silu(conv(x)) for the QK channels, channel-last (b, 2*qk, s). + silu_chan = group_id * qk_channels + head_id * HEAD_DIM + chan_off + silu_ptrs = ( + silu_save_ptr + + batch_id * silu_b_stride + + silu_chan[None, :] * silu_c_stride + + s_offs[:, None] * silu_s_stride + ) + tl.store( + silu_ptrs, + silu_out.to(silu_save_ptr.dtype.element_ty), + mask=s_mask[:, None], + ) + + norm_sq = tl.sum(silu_out * silu_out, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + out_typed = (silu_out * rstd[:, None]).to(out_ty) + + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + write_ptr = ( + qk_out_ptr + + group_id * qk_g_stride + + batch_id * qk_b_stride + + s_offs[:, None] * qk_s_stride + + v_head * qk_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + elif pid_row < R_qkv: + # ---- V: depthwise causal conv + silu (no l2norm, no repeat) ---- + local = pid_row - R_qk + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + chan = 2 * qk_channels + head_id * HEAD_DIM + chan_off + + if HAS_THD: + seq_start, seq_end = _mega_seq_bounds( + cu_seqlens_ptr, s_offs, seq_len, num_packed_seqs + ) + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + if HAS_THD: + x_mask = s_mask & (x_s >= seq_start) & (x_s < seq_end) + safe_x_s = tl.minimum(tl.maximum(x_s, 0), seq_len - 1) + else: + x_mask = (x_s >= 0) & (x_s < seq_len) + safe_x_s = x_s + x_ptr = ( + qkvzba_ptr + + safe_x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc = acc.to(out_ty).to(tl.float32) + silu_out = acc * tl.sigmoid(acc) + out_typed = silu_out.to(out_ty) + write_ptr = ( + value_ptr + + batch_id * v_b_stride + + s_offs[:, None] * v_s_stride + + head_id * v_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + elif pid_row < R_qkvz: + # ---- Z: copy qkvzba z slice into the final gate layout ---- + local = pid_row - R_qkv + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + z_chan = 2 * qk_channels + v_channels + head_id * HEAD_DIM + chan_off + src_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + z_chan[None, :] * qkvzba_c_stride + ) + z_val = tl.load(src_ptr, mask=s_mask[:, None]) + write_ptr = ( + gate_ptr + + batch_id * z_b_stride + + s_offs[:, None] * z_s_stride + + head_id * z_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, z_val, mask=s_mask[:, None]) + + else: + # ---- g/beta: -exp(A_log)*softplus(alpha+dt_bias) and sigmoid(beta) ---- + local = pid_row - R_qkvz + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + beta_chan = 2 * qk_channels + 2 * v_channels + head_id + alpha_chan = beta_chan + num_value_heads + + alpha_ptr = ( + qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + alpha_chan * qkvzba_c_stride + ) + beta_raw_ptr = ( + qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + beta_chan * qkvzba_c_stride + ) + alpha = tl.load(alpha_ptr, mask=s_mask, other=0.0).to(tl.float32) + beta_raw = tl.load(beta_raw_ptr, mask=s_mask, other=0.0).to(tl.float32) + A_log = tl.load(A_log_ptr + head_id).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + head_id).to(tl.float32) + + pre = alpha + dt_bias + softplus_val = tl.log(1.0 + tl.exp(pre)) + g = -tl.exp(A_log) * softplus_val + beta_sig = tl.sigmoid(beta_raw) + + g_store_ptr = ( + g_ptr + batch_id * g_b_stride + s_offs * g_s_stride + head_id * g_h_stride + ) + beta_store_ptr = ( + beta_ptr + + batch_id * beta_b_stride + + s_offs * beta_s_stride + + head_id * beta_h_stride + ) + tl.store(g_store_ptr, g.to(g_ptr.dtype.element_ty), mask=s_mask) + tl.store(beta_store_ptr, beta_sig.to(beta_ptr.dtype.element_ty), mask=s_mask) + + +# --------------------------------------------------------------------------- +# Forward orchestration +# --------------------------------------------------------------------------- + + +def _mega_pre_gated_delta_rule_forward( + qkvzba: Tensor, + conv1d_weight: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + cu_seqlens: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Single-launch mega forward. + + Returns ``(query, key, value, gate, beta, g, silu_qk_save)``; the last + element is the bf16-rounded QK ``silu(conv(x))`` laid out channel-last, + matching the streamed forward so the shared backward can consume it. + """ + + seq_len, batch, total_channels = qkvzba.shape + is_packed_thd = cu_seqlens is not None + num_packed_seqs = (cu_seqlens.shape[0] - 1) if is_packed_thd else 0 + + assert key_head_dim == value_head_dim, ( + "fused_mega_pre_gated_delta_rule currently requires " + f"key_head_dim == value_head_dim; got {key_head_dim=} {value_head_dim=}." + ) + assert _is_power_of_two(key_head_dim), ( + f"Mega kernel expects key_head_dim to be a power of two; got {key_head_dim=}." + ) + head_dim = key_head_dim + + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + repeat_factor = num_value_heads // num_key_heads + k_w = conv1d_weight.shape[-1] + + expected_channels = 2 * qk_channels + 2 * v_channels + 2 * num_value_heads + assert total_channels == expected_channels, ( + f"qkvzba last-dim mismatch: got {total_channels}, expected {expected_channels}." + ) + + out_dtype = qkvzba.dtype + device = qkvzba.device + + # Output buffers (identical layouts to the streamed path). + qk_out = torch.empty( + 2, batch, seq_len, num_value_heads, key_head_dim, dtype=out_dtype, device=device + ) + query = qk_out[0] + key = qk_out[1] + value = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + gate = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + g = torch.empty(batch, seq_len, num_value_heads, dtype=torch.float32, device=device) + beta = torch.empty(batch, seq_len, num_value_heads, dtype=out_dtype, device=device) + + # QK silu(conv(x)) persisted channel-last: (b, 2*qk_channels, s), stride(1)==1. + silu_qk_save = torch.empty( + (batch, seq_len, 2 * qk_channels), dtype=out_dtype, device=device + ).permute(0, 2, 1) + + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + + # Flat task-space row partition. + R_qk = batch * 2 * num_key_heads + R_v = batch * num_value_heads + R_z = batch * num_value_heads + R_gb = batch * num_value_heads + num_rows = R_qk + R_v + R_z + R_gb + + cu_seqlens_arg = cu_seqlens if is_packed_thd else qkvzba # dummy when dense + + grid = lambda meta: (num_rows, triton.cdiv(seq_len, meta["BLOCK_S"])) + _mega_forward_kernel[grid]( + qkvzba, + weight_2d, + A_log, + dt_bias, + qk_out, + value, + gate, + g, + beta, + silu_qk_save, + cu_seqlens_arg, + seq_len, + num_packed_seqs, + num_key_heads, + num_value_heads, + qk_channels, + v_channels, + R_qk, + R_v, + R_z, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + qk_out.stride(0), + qk_out.stride(1), + qk_out.stride(2), + qk_out.stride(3), + value.stride(0), + value.stride(1), + value.stride(2), + gate.stride(0), + gate.stride(1), + gate.stride(2), + g.stride(0), + g.stride(1), + g.stride(2), + beta.stride(0), + beta.stride(1), + beta.stride(2), + silu_qk_save.stride(0), + silu_qk_save.stride(1), + silu_qk_save.stride(2), + _L2NORM_EPS, + HEAD_DIM=head_dim, + K_W=k_w, + REPEAT=repeat_factor, + HAS_THD=is_packed_thd, + ) + + return query, key, value, gate, beta, g, silu_qk_save + + +# --------------------------------------------------------------------------- +# Backward kernel +# --------------------------------------------------------------------------- + + +def _mega_backward_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ] + + +@triton.autotune( + configs=_mega_backward_autotune_configs(), + key=["seq_len", "HEAD_DIM", "REPEAT", "num_key_heads", "num_value_heads"], + # The g/beta task atomic-adds per-head partials into these accumulators. + # reset_to_zero clears them before each autotune trial so trials don't stack. + reset_to_zero=["d_A_log_ptr", "d_dt_bias_ptr"], +) +@triton.jit +def _mega_backward_kernel( + # inputs + dq_ptr, + dk_ptr, + dv_ptr, + dgate_ptr, + dg_ptr, + dbeta_ptr, + silu_save_ptr, + qkvzba_ptr, + A_log_ptr, + dt_bias_ptr, + # outputs + d_silu_conv_ptr, + d_qkvzba_ptr, + d_A_log_ptr, + d_dt_bias_ptr, + # sizes / layout + seq_len, + num_key_heads, + num_value_heads, + qk_channels, + v_channels, + R_qk, + R_v, + R_z, + eps, + # dq / dk strides (b, s, h, d) + dq_b_stride, + dq_s_stride, + dq_h_stride, + dk_b_stride, + dk_s_stride, + dk_h_stride, + # dv strides (b, s, h, d) + dv_b_stride, + dv_s_stride, + dv_h_stride, + # dgate strides (b, s, h, d) + dgate_b_stride, + dgate_s_stride, + dgate_h_stride, + # dg / dbeta strides (b, s, h) + dg_b_stride, + dg_s_stride, + dg_h_stride, + dbeta_b_stride, + dbeta_s_stride, + dbeta_h_stride, + # silu_save strides (b, 2*qk, s) + silu_b_stride, + silu_c_stride, + silu_s_stride, + # qkvzba / d_qkvzba strides (s, b, C) + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + # d_silu_conv strides (b, conv_dim, s) + dsc_b_stride, + dsc_c_stride, + dsc_s_stride, + HEAD_DIM: tl.constexpr, + REPEAT: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """All-in-one backward for the QK / V / Z / g-beta branches. + + Mirrors the four streamed branch kernels in one flat task space. Conv input + and weight gradients are NOT produced here; the caller feeds ``d_silu_conv`` + into the external ``causal_conv1d_bwd_function`` exactly as the streamed + path does. Flat task space on ``program_id(0)``: + rows [0, R_qk) -> QK l2norm + repeat backward -> d_silu_conv[Q/K] + rows [R_qk, +R_v) -> V layout copy -> d_silu_conv[V] + rows [.., +R_z) -> Z layout copy -> d_qkvzba[z] + rows [.., end) -> g/beta chain rule -> d_qkvzba[alpha,beta], + atomic d_A_log/d_dt_bias + """ + + pid_row = tl.program_id(0) + pid_s = tl.program_id(1) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + chan_off = tl.arange(0, HEAD_DIM) + + R_qkv = R_qk + R_v + R_qkvz = R_qkv + R_z + + if pid_row < R_qk: + # ---- QK: repeat-reduce + l2norm backward -> d_silu_conv[Q/K] ---- + local = pid_row + heads_per_batch = 2 * num_key_heads + batch_id = local // heads_per_batch + lb = local - batch_id * heads_per_batch + group_id = lb // num_key_heads + head_id = lb - group_id * num_key_heads + is_query = group_id == 0 + is_key = group_id == 1 + chan = group_id * qk_channels + head_id * HEAD_DIM + chan_off + + d_normed = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + dq_ptrs = ( + dq_ptr + + batch_id * dq_b_stride + + s_offs[:, None] * dq_s_stride + + v_head * dq_h_stride + + chan_off[None, :] + ) + dk_ptrs = ( + dk_ptr + + batch_id * dk_b_stride + + s_offs[:, None] * dk_s_stride + + v_head * dk_h_stride + + chan_off[None, :] + ) + d_normed += tl.load(dq_ptrs, mask=s_mask[:, None] & is_query, other=0.0).to(tl.float32) + d_normed += tl.load(dk_ptrs, mask=s_mask[:, None] & is_key, other=0.0).to(tl.float32) + + silu_ptrs = ( + silu_save_ptr + + batch_id * silu_b_stride + + chan[None, :] * silu_c_stride + + s_offs[:, None] * silu_s_stride + ) + silu_bf16 = tl.load(silu_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + + norm_sq = tl.sum(silu_bf16 * silu_bf16, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + s_row = tl.sum(d_normed * silu_bf16, axis=1) + rstd3 = rstd * rstd * rstd + d_silu = rstd[:, None] * d_normed - rstd3[:, None] * silu_bf16 * s_row[:, None] + + dsc_ptrs = ( + d_silu_conv_ptr + + batch_id * dsc_b_stride + + chan[None, :] * dsc_c_stride + + s_offs[:, None] * dsc_s_stride + ) + tl.store(dsc_ptrs, d_silu.to(d_silu_conv_ptr.dtype.element_ty), mask=s_mask[:, None]) + + elif pid_row < R_qkv: + # ---- V: relayout dv -> d_silu_conv[V] ---- + local = pid_row - R_qk + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + dv_ptrs = ( + dv_ptr + + batch_id * dv_b_stride + + s_offs[:, None] * dv_s_stride + + head_id * dv_h_stride + + chan_off[None, :] + ) + dv_val = tl.load(dv_ptrs, mask=s_mask[:, None], other=0.0) + dsc_chan = 2 * qk_channels + head_id * HEAD_DIM + chan_off + dsc_ptrs = ( + d_silu_conv_ptr + + batch_id * dsc_b_stride + + dsc_chan[None, :] * dsc_c_stride + + s_offs[:, None] * dsc_s_stride + ) + tl.store(dsc_ptrs, dv_val, mask=s_mask[:, None]) + + elif pid_row < R_qkvz: + # ---- Z: relayout dgate -> d_qkvzba[z] ---- + local = pid_row - R_qkv + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + dgate_ptrs = ( + dgate_ptr + + batch_id * dgate_b_stride + + s_offs[:, None] * dgate_s_stride + + head_id * dgate_h_stride + + chan_off[None, :] + ) + dgate_val = tl.load(dgate_ptrs, mask=s_mask[:, None], other=0.0) + dz_chan = 2 * qk_channels + v_channels + head_id * HEAD_DIM + chan_off + dz_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + dz_chan[None, :] * qkvzba_c_stride + ) + tl.store(dz_ptrs, dgate_val, mask=s_mask[:, None]) + + else: + # ---- g/beta: chain rule -> d_qkvzba[alpha,beta] + atomic d_A_log/d_dt_bias ---- + local = pid_row - R_qkvz + batch_id = local // num_value_heads + head_id = local - batch_id * num_value_heads + beta_chan = 2 * qk_channels + 2 * v_channels + head_id + alpha_chan = beta_chan + num_value_heads + + alpha_ptrs = ( + qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + alpha_chan * qkvzba_c_stride + ) + beta_ptrs = ( + qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + beta_chan * qkvzba_c_stride + ) + alpha = tl.load(alpha_ptrs, mask=s_mask, other=0.0).to(tl.float32) + beta_raw = tl.load(beta_ptrs, mask=s_mask, other=0.0).to(tl.float32) + A_log = tl.load(A_log_ptr + head_id).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + head_id).to(tl.float32) + + pre = alpha + dt_bias + sigmoid_pre = tl.sigmoid(pre) + softplus_pre = tl.log(1.0 + tl.exp(pre)) + exp_A = tl.exp(A_log) + g = -exp_A * softplus_pre + beta_sig = tl.sigmoid(beta_raw) + + dg_ptrs = ( + dg_ptr + batch_id * dg_b_stride + s_offs * dg_s_stride + head_id * dg_h_stride + ) + dbeta_ptrs = ( + dbeta_ptr + + batch_id * dbeta_b_stride + + s_offs * dbeta_s_stride + + head_id * dbeta_h_stride + ) + d_g = tl.load(dg_ptrs, mask=s_mask, other=0.0).to(tl.float32) + d_beta_out = tl.load(dbeta_ptrs, mask=s_mask, other=0.0).to(tl.float32) + + d_alpha = d_g * (-exp_A * sigmoid_pre) + d_beta_raw = d_beta_out * beta_sig * (1.0 - beta_sig) + + d_alpha_ptrs = ( + d_qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + alpha_chan * qkvzba_c_stride + ) + d_beta_ptrs = ( + d_qkvzba_ptr + + s_offs * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + beta_chan * qkvzba_c_stride + ) + tl.store(d_alpha_ptrs, d_alpha.to(d_qkvzba_ptr.dtype.element_ty), mask=s_mask) + tl.store(d_beta_ptrs, d_beta_raw.to(d_qkvzba_ptr.dtype.element_ty), mask=s_mask) + + d_g_masked = tl.where(s_mask, d_g, 0.0) + d_alpha_masked = tl.where(s_mask, d_alpha, 0.0) + d_A_log_partial = tl.sum(d_g_masked * g) + d_dt_bias_partial = tl.sum(d_alpha_masked) + tl.atomic_add(d_A_log_ptr + head_id, d_A_log_partial) + tl.atomic_add(d_dt_bias_ptr + head_id, d_dt_bias_partial) + + +# --------------------------------------------------------------------------- +# Backward orchestration +# --------------------------------------------------------------------------- + + +def _mega_pre_gated_delta_rule_backward( + qkvzba: Tensor, + conv1d_weight: Tensor, + silu_qk_save: Tensor, + dq: Tensor, + dk: Tensor, + dv: Tensor, + dgate: Tensor, + dbeta: Tensor, + dg: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Two-launch mega backward: one fused branch kernel + external conv bwd. + + Collapses the four streamed branch kernels (QK l2norm/repeat, V layout, Z + layout, g/beta chain rule) into a single Triton launch, then delegates the + depthwise conv input/weight gradients to ``causal_conv1d_bwd_function`` as + the streamed path does. Returns ``(d_qkvzba, d_weight, d_A_log, d_dt_bias)``. + """ + + seq_len, batch, _ = qkvzba.shape + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + conv_dim = 2 * qk_channels + v_channels + k_w = conv1d_weight.shape[-1] + device = qkvzba.device + head_dim = key_head_dim + + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + # Channel-last conv input view (stride(1)==1) — no copy. + qkvzba_conv = qkvzba[:, :, :conv_dim].permute(1, 2, 0) + + # d_silu_conv channel-last (b, conv_dim, s), stride(1)==1. + d_silu_conv = torch.empty( + (batch, seq_len, conv_dim), dtype=qkvzba.dtype, device=device + ).permute(0, 2, 1) + d_qkvzba = torch.empty_like(qkvzba) + d_A_log_fp32 = torch.zeros(num_value_heads, dtype=torch.float32, device=device) + d_dt_bias_fp32 = torch.zeros(num_value_heads, dtype=torch.float32, device=device) + + R_qk = batch * 2 * num_key_heads + R_v = batch * num_value_heads + R_z = batch * num_value_heads + R_gb = batch * num_value_heads + num_rows = R_qk + R_v + R_z + R_gb + + grid = lambda meta: (num_rows, triton.cdiv(seq_len, meta["BLOCK_S"])) + _mega_backward_kernel[grid]( + dq, + dk, + dv, + dgate, + dg, + dbeta, + silu_qk_save, + qkvzba, + A_log, + dt_bias, + d_silu_conv, + d_qkvzba, + d_A_log_fp32, + d_dt_bias_fp32, + seq_len, + num_key_heads, + num_value_heads, + qk_channels, + v_channels, + R_qk, + R_v, + R_z, + _L2NORM_EPS, + dq.stride(0), + dq.stride(1), + dq.stride(2), + dk.stride(0), + dk.stride(1), + dk.stride(2), + dv.stride(0), + dv.stride(1), + dv.stride(2), + dgate.stride(0), + dgate.stride(1), + dgate.stride(2), + dg.stride(0), + dg.stride(1), + dg.stride(2), + dbeta.stride(0), + dbeta.stride(1), + dbeta.stride(2), + silu_qk_save.stride(0), + silu_qk_save.stride(1), + silu_qk_save.stride(2), + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + d_silu_conv.stride(0), + d_silu_conv.stride(1), + d_silu_conv.stride(2), + HEAD_DIM=head_dim, + REPEAT=num_value_heads // num_key_heads, + ) + + # External conv backward: writes d_x into d_qkvzba's conv slice (strided + # view, no copy) and returns d_weight. Same call shape as the streamed path. + seq_stride = qkvzba.stride(0) + batch_stride = qkvzba.stride(1) + d_x_conv_view = d_qkvzba.as_strided( + (batch, conv_dim, seq_len), + (batch_stride, 1, seq_stride), + ) + _, d_weight_fp32, _, _ = _causal_conv1d_bwd_function( + qkvzba_conv, + weight_2d, + None, # no bias + d_silu_conv, + seq_idx, + None, # initial_states + None, # dfinal_states + d_x_conv_view, # dx pre-allocated into d_qkvzba's conv slice + False, # return_dinitial_states + True, # activation (silu folded into conv bwd) + ) + + d_weight = d_weight_fp32.view(*conv1d_weight.shape).to(conv1d_weight.dtype) + d_A_log = d_A_log_fp32.to(A_log.dtype) + d_dt_bias = d_dt_bias_fp32.to(dt_bias.dtype) + return d_qkvzba, d_weight, d_A_log, d_dt_bias + + +# --------------------------------------------------------------------------- +# Autograd wiring +# --------------------------------------------------------------------------- + + +class _FusedMegaPreGatedDeltaRuleFunction(torch.autograd.Function): + """Autograd entry point for the mega path. + + Forward dispatches to the single-launch mega forward. Backward currently + reuses the streamed conv-backend-delegated backward (which consumes the + same saved ``silu_qk_save`` layout); a dedicated mega backward is layered + in behind this same entry point. + """ + + @staticmethod + def forward( + ctx, + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ): + ctx.num_key_heads = num_key_heads + ctx.num_value_heads = num_value_heads + ctx.key_head_dim = key_head_dim + ctx.value_head_dim = value_head_dim + query, key, value, gate, beta, g, silu_qk_save = ( + _mega_pre_gated_delta_rule_forward( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + cu_seqlens=cu_seqlens, + ) + ) + ctx.has_seq_idx = seq_idx is not None + if ctx.has_seq_idx: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx) + else: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save) + return query, key, value, gate, beta, g + + @staticmethod + def backward(ctx, dq, dk, dv, dgate, dbeta, dg): + if ctx.has_seq_idx: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx = ctx.saved_tensors + else: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save = ctx.saved_tensors + seq_idx = None + d_qkvzba, d_weight, d_A_log, d_dt_bias = _mega_pre_gated_delta_rule_backward( + qkvzba, + conv1d_weight, + silu_qk_save, + dq, + dk, + dv, + dgate, + dbeta, + dg, + A_log, + dt_bias, + num_key_heads=ctx.num_key_heads, + num_value_heads=ctx.num_value_heads, + key_head_dim=ctx.key_head_dim, + value_head_dim=ctx.value_head_dim, + seq_idx=seq_idx, + ) + return (d_qkvzba, d_weight, d_A_log, d_dt_bias, None, None, None, None, None, None) + + +def fused_mega_pre_gated_delta_rule( + qkvzba: Tensor, + conv1d_weight: Tensor, + conv1d_bias: Optional[Tensor], + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + use_qk_l2norm: bool = True, + cu_seqlens: Optional[Tensor] = None, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Mega fused pre-gated-delta-rule entry point. + + Args: + qkvzba: ``[seq_len, batch, in_proj_dim]`` projection output. + conv1d_weight: ``[conv_dim, 1, k_w]`` depthwise conv weight. + conv1d_bias: Must be ``None`` in the mega path. + A_log: ``[num_value_heads]`` raw decay parameter. + dt_bias: ``[num_value_heads]`` time-step bias. + num_key_heads / num_value_heads / key_head_dim / value_head_dim: GDN + architecture parameters. ``num_value_heads`` must be a multiple of + ``num_key_heads`` and ``key_head_dim == value_head_dim``. + use_qk_l2norm: Must be ``True`` for parity with the streamed path. + cu_seqlens: Optional packed THD cumulative sequence lengths. + seq_idx: Optional precomputed token-to-sequence map for packed THD mode. + + Returns: + ``(query, key, value, gate, beta, g)`` matching the unfused and streamed + fused pre-GDR APIs. + """ + + assert qkvzba.is_cuda, ( + "fused_mega_pre_gated_delta_rule requires CUDA inputs; " + f"got qkvzba.device={qkvzba.device}." + ) + assert conv1d_bias is None, ( + "Conv bias is not supported by fused_mega_pre_gated_delta_rule " + "(production GDN config has none)." + ) + assert use_qk_l2norm, ( + "use_qk_l2norm=False is not supported by fused_mega_pre_gated_delta_rule " + "(the backward closes over the l2norm path)." + ) + assert num_value_heads % num_key_heads == 0, ( + f"{num_value_heads=} must be a multiple of {num_key_heads=}." + ) + assert key_head_dim == value_head_dim, ( + "fused_mega_pre_gated_delta_rule currently requires " + f"key_head_dim == value_head_dim; got {key_head_dim=} {value_head_dim=}." + ) + if cu_seqlens is not None: + assert cu_seqlens.is_cuda, ( + "Packed fused_mega_pre_gated_delta_rule requires CUDA cu_seqlens; " + f"got cu_seqlens.device={cu_seqlens.device}." + ) + assert cu_seqlens.dtype == torch.int32, ( + "Packed fused_mega_pre_gated_delta_rule requires int32 cu_seqlens; " + f"got {cu_seqlens.dtype=}." + ) + assert cu_seqlens.dim() == 1, ( + "Packed fused_mega_pre_gated_delta_rule expects 1-D cu_seqlens; " + f"got {cu_seqlens.shape=}." + ) + assert qkvzba.shape[1] == 1, ( + "Packed THD fused_mega_pre_gated_delta_rule expects batch dimension 1; " + f"got qkvzba.shape={qkvzba.shape}." + ) + assert cu_seqlens.shape[0] >= 2, ( + "Packed fused_mega_pre_gated_delta_rule requires at least one packed sequence; " + f"got {cu_seqlens.shape=}." + ) + assert cu_seqlens[0].item() == 0, ( + "Packed fused_mega_pre_gated_delta_rule requires cu_seqlens[0] == 0, " + f"got {cu_seqlens[0].item()}." + ) + assert cu_seqlens[-1].item() == qkvzba.shape[0], ( + "Packed fused_mega_pre_gated_delta_rule requires cu_seqlens[-1] to match " + f"seq_len, got {cu_seqlens[-1].item()} vs {qkvzba.shape[0]}." + ) + cu_seqlens = cu_seqlens.contiguous() + seq_idx = _resolve_packed_seq_idx(cu_seqlens, seq_idx, qkvzba.shape[0]) + else: + assert seq_idx is None, "seq_idx requires cu_seqlens for packed THD mode." + + return _FusedMegaPreGatedDeltaRuleFunction.apply( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ) diff --git a/megatron/core/fusions/fused_pre_gated_delta_rule.py b/megatron/core/fusions/fused_pre_gated_delta_rule.py new file mode 100644 index 00000000000..a7cf7a92a72 --- /dev/null +++ b/megatron/core/fusions/fused_pre_gated_delta_rule.py @@ -0,0 +1,2179 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Fused pre-gated-delta-rule projection kernels. + +The public entry point consumes the dense ``qkvzba`` projection and returns +``query``, ``key``, ``value``, ``gate``, ``beta``, and ``g`` in the layouts +expected by the gated delta rule. The forward path keeps QK, V, Z, and +G/Beta as separate streamed scopes. The backward mirrors those scopes for +layout/l2norm/g-beta work, then delegates depthwise conv gradients to the +``causal_conv1d`` backend. + +Unsupported cases are rejected at the Python entry point: CPU tensors, +conv bias, and ``use_qk_l2norm=False``. Packed THD sequences use separate +QK/V causal-conv kernels so the dense BSHD kernels stay free of packed +metadata and runtime branches. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +# The 1.6.1+ ``causal_conv1d`` package exposes the lower-level binding via +# ``causal_conv1d.cpp_functions.causal_conv1d_bwd_function``; older builds +# (still common in some older environments) expose the same +# function under ``causal_conv1d_cuda.causal_conv1d_bwd``. Try both so the +# fast path is taken everywhere the package is installed. +from torch import Tensor + +try: + from causal_conv1d.cpp_functions import ( + causal_conv1d_bwd_function as _causal_conv1d_bwd_function, + ) +except ImportError: + import causal_conv1d_cuda as _causal_conv1d_cuda + + _causal_conv1d_bwd_function = _causal_conv1d_cuda.causal_conv1d_bwd + + +_L2NORM_EPS = 1e-6 + +_QK_STREAM_SLOT = 0 +_V_STREAM_SLOT = 2 +_G_BETA_STREAM_SLOT = 3 +_Z_STREAM_SLOT = 4 + +_LAYOUT_BLOCK_S = 64 + + +# --------------------------------------------------------------------------- +# Forward kernels +# --------------------------------------------------------------------------- + + +def _conv_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=4), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ] + + +def _g_beta_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 32, "BLOCK_H": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64, "BLOCK_H": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64, "BLOCK_H": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 16}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 256, "BLOCK_H": 16}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 256, "BLOCK_H": 32}, num_warps=8, num_stages=2), + ] + + +@triton.autotune( + configs=_conv_autotune_configs(), + key=["seq_len", "HEAD_DIM", "K_W", "APPLY_L2", "REPEAT", "NUM_GROUPS"], +) +@triton.jit +def _conv_silu_project_kernel( + qkvzba_ptr, + weight_ptr, + bias_ptr, + out_ptr, + silu_save_ptr, + seq_len, + num_in_heads, + in_channel_offset, + in_group_stride, + silu_save_chan_offset, + silu_save_group_stride, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + bias_stride, + out_group_dim_stride, + out_b_stride, + out_s_stride, + out_h_stride, + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + eps, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + NUM_GROUPS: tl.constexpr, + HAS_BIAS: tl.constexpr, + APPLY_L2: tl.constexpr, + SAVE_SILU: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Depthwise conv1d + silu + (optional l2norm) + (optional head repeat). + + Grid layout (program_id): + 0: batch * NUM_GROUPS * num_in_heads (flat) + 1: num_seq_blocks + + Args: + in_channel_offset: starting channel index of the first group inside + ``qkvzba``. 0 for QK, ``v_channel_offset`` for V. + in_group_stride: channel distance between logical groups. For QK this + is ``qk_channels`` so group 0 is Q and group 1 is K. For V this is + 0 because ``NUM_GROUPS == 1``. + out_group_dim_stride: output-storage distance between logical groups. QK + passes a grouped output buffer and V passes 0. + """ + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_in_heads * NUM_GROUPS + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_in_heads + head_id = local_bgh - group_id * num_in_heads + + chan_off = tl.arange(0, HEAD_DIM) + group_channel_offset = in_channel_offset + group_id * in_group_stride + chan = group_channel_offset + head_id * HEAD_DIM + chan_off + + if HAS_BIAS: + bias = tl.load(bias_ptr + chan * bias_stride).to(tl.float32) + else: + bias = tl.zeros([HEAD_DIM], dtype=tl.float32) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask = (x_s >= 0) & (x_s < seq_len) + x_ptr = ( + qkvzba_ptr + + x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc += bias[None, :] + # Mimic the unfused F.conv1d rounding: the reference path stores the conv + # output in the input dtype (bf16) before silu, so do the same here. This + # keeps the fused output bit-aligned with the reference within one ULP. + acc = acc.to(out_ptr.dtype.element_ty).to(tl.float32) + silu_out = acc * tl.sigmoid(acc) + + if APPLY_L2: + # F.silu rounds to the input dtype before l2norm reads it. Round-trip + # via bf16 to match that precision. + silu_out = silu_out.to(out_ptr.dtype.element_ty).to(tl.float32) + if SAVE_SILU: + # Persist only the QK silu output in the channel-last layout + # consumed by the QK l2norm backward. + silu_save_chan = ( + silu_save_chan_offset + + group_id * silu_save_group_stride + + head_id * HEAD_DIM + + chan_off + ) + silu_save_ptrs = ( + silu_save_ptr + + batch_id * silu_save_b_stride + + silu_save_chan[None, :] * silu_save_c_stride + + s_offs[:, None] * silu_save_s_stride + ) + tl.store( + silu_save_ptrs, + silu_out.to(silu_save_ptr.dtype.element_ty), + mask=s_mask[:, None], + ) + norm_sq = tl.sum(silu_out * silu_out, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + out = silu_out * rstd[:, None] + else: + # No l2norm follows. The final store→bf16 already does the rounding; + # an intermediate bf16 round-trip would be redundant. + out = silu_out + + out_typed = out.to(out_ptr.dtype.element_ty) + + # Write the same data to ``REPEAT`` adjacent value heads. ``REPEAT == 1`` + # is the no-repeat case (V branch is handled by a separate kernel that + # always has REPEAT == 1, but using the same code here is convenient). + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + write_ptr = ( + out_ptr + + group_id * out_group_dim_stride + + batch_id * out_b_stride + + s_offs[:, None] * out_s_stride + + v_head * out_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + +@triton.jit +def _thd_seq_bounds(cu_seqlens_ptr, token_offsets, total_tokens, num_packed_seqs): + """Return lane-wise packed sequence bounds for flattened THD tokens.""" + + safe_tokens = tl.minimum(token_offsets, total_tokens - 1) + seq_start = token_offsets * 0 + seq_end = token_offsets * 0 + total_tokens + + seq_id = 0 + while seq_id < num_packed_seqs: + start = tl.load(cu_seqlens_ptr + seq_id) + end = tl.load(cu_seqlens_ptr + seq_id + 1) + in_seq = (safe_tokens >= start) & (safe_tokens < end) + seq_start = tl.where(in_seq, start, seq_start) + seq_end = tl.where(in_seq, end, seq_end) + seq_id += 1 + + return seq_start, seq_end + + +@triton.autotune( + configs=_conv_autotune_configs(), + key=["seq_len", "HEAD_DIM", "K_W", "APPLY_L2", "REPEAT", "NUM_GROUPS"], +) +@triton.jit +def _conv_silu_project_thd_kernel( + qkvzba_ptr, + weight_ptr, + bias_ptr, + out_ptr, + silu_save_ptr, + cu_seqlens_ptr, + seq_len, + num_packed_seqs, + num_in_heads, + in_channel_offset, + in_group_stride, + silu_save_chan_offset, + silu_save_group_stride, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + bias_stride, + out_group_dim_stride, + out_b_stride, + out_s_stride, + out_h_stride, + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + eps, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + NUM_GROUPS: tl.constexpr, + HAS_BIAS: tl.constexpr, + APPLY_L2: tl.constexpr, + SAVE_SILU: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """THD depthwise conv1d + silu + optional l2norm/repeat. + + This is intentionally separate from ``_conv_silu_project_kernel`` so + packed sequence boundary metadata never enters the dense BSHD hot path. + Only the causal-conv loads use ``cu_seqlens``; the following per-token + transforms and stores are identical to the dense path. + """ + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_in_heads * NUM_GROUPS + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_in_heads + head_id = local_bgh - group_id * num_in_heads + + chan_off = tl.arange(0, HEAD_DIM) + group_channel_offset = in_channel_offset + group_id * in_group_stride + chan = group_channel_offset + head_id * HEAD_DIM + chan_off + + if HAS_BIAS: + bias = tl.load(bias_ptr + chan * bias_stride).to(tl.float32) + else: + bias = tl.zeros([HEAD_DIM], dtype=tl.float32) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + seq_start, seq_end = _thd_seq_bounds(cu_seqlens_ptr, s_offs, seq_len, num_packed_seqs) + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask = s_mask & (x_s >= seq_start) & (x_s < seq_end) + safe_x_s = tl.minimum(tl.maximum(x_s, 0), seq_len - 1) + x_ptr = ( + qkvzba_ptr + + safe_x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc += bias[None, :] + acc = acc.to(out_ptr.dtype.element_ty).to(tl.float32) + silu_out = acc * tl.sigmoid(acc) + + if APPLY_L2: + silu_out = silu_out.to(out_ptr.dtype.element_ty).to(tl.float32) + if SAVE_SILU: + silu_save_chan = ( + silu_save_chan_offset + + group_id * silu_save_group_stride + + head_id * HEAD_DIM + + chan_off + ) + silu_save_ptrs = ( + silu_save_ptr + + batch_id * silu_save_b_stride + + silu_save_chan[None, :] * silu_save_c_stride + + s_offs[:, None] * silu_save_s_stride + ) + tl.store( + silu_save_ptrs, + silu_out.to(silu_save_ptr.dtype.element_ty), + mask=s_mask[:, None], + ) + norm_sq = tl.sum(silu_out * silu_out, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + out = silu_out * rstd[:, None] + else: + out = silu_out + + out_typed = out.to(out_ptr.dtype.element_ty) + + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + write_ptr = ( + out_ptr + + group_id * out_group_dim_stride + + batch_id * out_b_stride + + s_offs[:, None] * out_s_stride + + v_head * out_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + +@triton.jit +def _copy_z_kernel( + qkvzba_ptr, + gate_ptr, + seq_len, + num_v_heads, + z_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + gate_b_stride, + gate_s_stride, + gate_h_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Copy the z slice from qkvzba into the final gate layout.""" + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + z_chan = z_channel_offset + head_id * HEAD_DIM + chan_off + z_src_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + z_chan[None, :] * qkvzba_c_stride + ) + z_val = tl.load(z_src_ptr, mask=s_mask[:, None]) + z_write_ptr = ( + gate_ptr + + batch_id * gate_b_stride + + s_offs[:, None] * gate_s_stride + + head_id * gate_h_stride + + chan_off[None, :] + ) + tl.store(z_write_ptr, z_val, mask=s_mask[:, None]) + + +@triton.autotune(configs=_g_beta_autotune_configs(), key=["seq_len", "num_v_heads"]) +@triton.jit +def _compute_g_and_beta_kernel( + qkvzba_ptr, + A_log_ptr, + dt_bias_ptr, + g_out_ptr, + beta_out_ptr, + seq_len, + num_v_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + g_b_stride, + g_s_stride, + g_h_stride, + beta_b_stride, + beta_s_stride, + beta_h_stride, + BLOCK_S: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """Compute ``g = -exp(A_log) * softplus(alpha + dt_bias)`` and ``sigmoid(beta)``.""" + + pid_b = tl.program_id(0) + pid_s = tl.program_id(1) + pid_h = tl.program_id(2) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + s_mask = s_offs < seq_len + h_mask = h_offs < num_v_heads + mask = s_mask[:, None] & h_mask[None, :] + + alpha_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + beta_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + + alpha = tl.load(alpha_ptr, mask=mask, other=0.0).to(tl.float32) + beta = tl.load(beta_ptr, mask=mask, other=0.0).to(tl.float32) + + A_log = tl.load(A_log_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + + pre = alpha + dt_bias[None, :] + # softplus(x) = log(1 + exp(x)); torch's softplus thresholds at x>20 but we + # rely on fp32 evaluation here, which stays well within range for typical + # GDN inputs (the unfused path computes the same expression). + softplus_val = tl.log(1.0 + tl.exp(pre)) + g = -tl.exp(A_log)[None, :] * softplus_val + beta_sig = tl.sigmoid(beta) + + g_ptr = ( + g_out_ptr + + pid_b * g_b_stride + + s_offs[:, None] * g_s_stride + + h_offs[None, :] * g_h_stride + ) + beta_out_ptr_calc = ( + beta_out_ptr + + pid_b * beta_b_stride + + s_offs[:, None] * beta_s_stride + + h_offs[None, :] * beta_h_stride + ) + tl.store(g_ptr, g.to(g_out_ptr.dtype.element_ty), mask=mask) + tl.store(beta_out_ptr_calc, beta_sig.to(beta_out_ptr.dtype.element_ty), mask=mask) + + +# --------------------------------------------------------------------------- +# Backward kernels +# --------------------------------------------------------------------------- + + +@triton.jit +def _conv_silu_l2norm_backward_kernel( + qkvzba_ptr, + weight_ptr, + d_out_ptr, + d_qkvzba_ptr, + d_w_partial_ptr, + seq_len, + num_qk_heads, + in_channel_offset, + eps, + d_out_scale, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + d_out_b_stride, + d_out_s_stride, + d_out_h_stride, + d_wp_b_stride, + d_wp_h_stride, + d_wp_s_stride, + d_wp_c_stride, + d_wp_w_stride, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + BLOCK_S: tl.constexpr, + USE_L2NORM: tl.constexpr, + V_HEAD_SHARED: tl.constexpr, +): + """Backward for the Q / K / V branches of ``_conv_silu_project_kernel``. + + ``USE_L2NORM`` is a constexpr branch: ``True`` for the QK branches (with + l2norm) and ``False`` for the V branch. The V case skips the l2norm + intermediates entirely — Triton DCE drops them at compile time. The + ``REPEAT=2`` workaround for the channel-collapse codegen bug still + applies in both branches. + + Forward (no bias, with l2norm, with REPEAT-way head broadcast): + acc = depthwise_conv(qkvzba_qk_slice, weight_qk_slice) + acc_bf16 = acc.to(bf16).to(fp32) # F.conv1d rounding + silu_out = acc_bf16 * sigmoid(acc_bf16) + silu_bf16 = silu_out.to(bf16).to(fp32) # round before l2norm + norm_sq = sum_c silu_bf16^2 + rstd = 1 / sqrt(norm_sq + eps) + out = silu_bf16 * rstd + # out is stored identically to REPEAT adjacent value heads. + + Backward (given d_out for each v_head): + d_qk_out = Σ_{r in REPEAT} d_v_out[head_id * REPEAT + r] + S = Σ_c d_qk_out_c * silu_bf16_c + d_silu_c = rstd * d_qk_out_c - rstd^3 * silu_bf16_c * S + d_acc_c = d_silu_c * silu'(acc_bf16) + d_w[c, i] = Σ_{b, t} d_acc[t, c] * x[t + i - (K_W - 1), c] + d_x[u, c] += Σ_i d_acc[u + (K_W - 1) - i, c] * w[c, i] + + ``d_w`` uses the per-program partial-buffer pattern (see the V backward + kernel). ``d_qkvzba`` uses bf16 atomic_add because the K_W − 1 boundary + input rows cross seq-block programs. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_qk_heads + head_id = pid_bh - batch_id * num_qk_heads + + chan_off = tl.arange(0, HEAD_DIM) + chan = in_channel_offset + head_id * HEAD_DIM + chan_off + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # ----- Forward recompute (conv + silu + l2norm) ----- + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask = (x_s >= 0) & (x_s < seq_len) + x_ptr = ( + qkvzba_ptr + + x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + acc += w_tap[None, :] * x_val + acc = acc.to(d_qkvzba_ptr.dtype.element_ty).to(tl.float32) + + # ----- Sum d_out across REPEAT v_heads ----- + # For QK: v_head = head_id*REPEAT + r, summing REPEAT distinct heads. + # For V (V_HEAD_SHARED=True): both r iterations load the SAME v_head + # (head_id), so d_qk_out = REPEAT * d_value[head_id]; the host passes + # d_out_scale = 1/REPEAT to recover d_value[head_id]. The duplicate + # load goes through L2, so the kernel-side cost is roughly one load; + # the trick was needed to avoid the REPEAT=1 codegen bug without + # having to allocate a doubled d_value tensor on the host. + d_qk_out = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for r in tl.static_range(REPEAT): + if V_HEAD_SHARED: + v_head = head_id + else: + v_head = head_id * REPEAT + r + d_out_ptrs = ( + d_out_ptr + + batch_id * d_out_b_stride + + s_offs[:, None] * d_out_s_stride + + v_head * d_out_h_stride + + chan_off[None, :] + ) + d_qk_out += tl.load(d_out_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + d_qk_out = d_qk_out * d_out_scale + + # ----- l2norm backward gated by USE_L2NORM constexpr. ----- + if USE_L2NORM: + silu_out = acc * tl.sigmoid(acc) + silu_bf16 = silu_out.to(d_qkvzba_ptr.dtype.element_ty).to(tl.float32) + norm_sq = tl.sum(silu_bf16 * silu_bf16, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + s_row = tl.sum(d_qk_out * silu_bf16, axis=1) + rstd3 = rstd * rstd * rstd + d_silu = rstd[:, None] * d_qk_out - rstd3[:, None] * silu_bf16 * s_row[:, None] + else: + d_silu = d_qk_out + + # ----- silu backward ----- + sig_acc = tl.sigmoid(acc) + silu_prime = sig_acc + acc * sig_acc * (1.0 - sig_acc) + d_acc = d_silu * silu_prime + d_acc = tl.where(s_mask[:, None], d_acc, 0.0) + + # ----- d_w via per-program partial, d_x via atomic_add ----- + partial_base = ( + d_w_partial_ptr + + batch_id * d_wp_b_stride + + head_id * d_wp_h_stride + + pid_s * d_wp_s_stride + ) + + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask_inner = (x_s >= 0) & (x_s < seq_len) + x_ptr = ( + qkvzba_ptr + + x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask_inner[:, None], other=0.0).to(tl.float32) + d_w_partial = tl.sum(d_acc * x_val, axis=0) + tl.store( + partial_base + chan_off * d_wp_c_stride + i * d_wp_w_stride, + d_w_partial, + ) + + w_tap = tl.load( + weight_ptr + chan * weight_c_stride + i * weight_w_stride + ).to(tl.float32) + contribution = d_acc * w_tap[None, :] + d_qkvzba_target = ( + d_qkvzba_ptr + + x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + tl.atomic_add( + d_qkvzba_target, + contribution.to(d_qkvzba_ptr.dtype.element_ty), + mask=x_mask_inner[:, None], + ) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ], + key=["seq_len", "HEAD_DIM", "REPEAT"], +) +@triton.jit +def _l2norm_repeat_backward_kernel( + d_qk_out_ptr, # (b, s, num_v_heads, head_dim) — gradient from downstream + silu_bf16_ptr, # (b, conv_dim, s) — silu(conv(x)) recomputed for QK channels + d_silu_bf16_ptr, # (b, conv_dim, s) — output gradient w.r.t. silu(conv(x)) + seq_len, + num_qk_heads, + channel_offset, # 0 for Q, qk_channels for K — indexes into conv_dim + eps, + d_qk_b_stride, + d_qk_s_stride, + d_qk_h_stride, + silu_b_stride, + silu_c_stride, + silu_s_stride, + d_silu_b_stride, + d_silu_c_stride, + d_silu_s_stride, + HEAD_DIM: tl.constexpr, + REPEAT: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """l2norm + REPEAT-way head broadcast backward. + + Forward (per QK head): + silu_bf16 ∈ R^{HEAD_DIM} # silu(conv(x)) rounded to bf16 + norm_sq = Σ_c silu_bf16_c^2 + rstd = 1 / sqrt(norm_sq + eps) + out = silu_bf16 * rstd + # out is broadcast identically to REPEAT adjacent v_heads. + + Backward (given d_qk_out for each v_head): + d_normed = Σ_{r in REPEAT} d_qk_out[head_id * REPEAT + r] + S = Σ_c d_normed_c * silu_bf16_c + d_silu_c = rstd * d_normed_c - rstd^3 * silu_bf16_c * S + + The output ``d_silu_bf16`` is the gradient w.r.t. ``silu(conv(x))`` — + exactly what ``causal_conv1d_bwd_function`` consumes as its ``dout`` + argument when ``activation="silu"`` is in effect on the forward. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_qk_heads + head_id = pid_bh - batch_id * num_qk_heads + + chan_off = tl.arange(0, HEAD_DIM) + chan = channel_offset + head_id * HEAD_DIM + chan_off + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # ----- Sum d_qk_out across REPEAT v_heads ----- + d_normed = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + d_out_ptrs = ( + d_qk_out_ptr + + batch_id * d_qk_b_stride + + s_offs[:, None] * d_qk_s_stride + + v_head * d_qk_h_stride + + chan_off[None, :] + ) + d_normed += tl.load(d_out_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + + # ----- Load silu_bf16 ----- + silu_ptrs = ( + silu_bf16_ptr + + batch_id * silu_b_stride + + chan[None, :] * silu_c_stride + + s_offs[:, None] * silu_s_stride + ) + silu_bf16 = tl.load(silu_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + + # ----- l2norm backward ----- + norm_sq = tl.sum(silu_bf16 * silu_bf16, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + s_row = tl.sum(d_normed * silu_bf16, axis=1) + rstd3 = rstd * rstd * rstd + d_silu = rstd[:, None] * d_normed - rstd3[:, None] * silu_bf16 * s_row[:, None] + + # ----- Store d_silu (same (b, conv_dim, s) layout as silu_bf16_ptr) ----- + d_silu_ptrs = ( + d_silu_bf16_ptr + + batch_id * d_silu_b_stride + + chan[None, :] * d_silu_c_stride + + s_offs[:, None] * d_silu_s_stride + ) + tl.store(d_silu_ptrs, d_silu.to(d_silu_bf16_ptr.dtype.element_ty), mask=s_mask[:, None]) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ], + key=["seq_len", "HEAD_DIM", "REPEAT"], +) +@triton.jit +def _qk_l2norm_repeat_backward_kernel( + dq_ptr, + dk_ptr, + silu_bf16_ptr, + d_silu_bf16_ptr, + seq_len, + num_qk_heads, + qk_channels, + eps, + dq_b_stride, + dq_s_stride, + dq_h_stride, + dk_b_stride, + dk_s_stride, + dk_h_stride, + silu_b_stride, + silu_c_stride, + silu_s_stride, + d_silu_b_stride, + d_silu_c_stride, + d_silu_s_stride, + HEAD_DIM: tl.constexpr, + REPEAT: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Merged Q/K l2norm + REPEAT-way head broadcast backward.""" + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_qk_heads * 2 + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_qk_heads + head_id = local_bgh - group_id * num_qk_heads + is_query = group_id == 0 + is_key = group_id == 1 + + chan_off = tl.arange(0, HEAD_DIM) + chan = group_id * qk_channels + head_id * HEAD_DIM + chan_off + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + d_normed = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + dq_ptrs = ( + dq_ptr + + batch_id * dq_b_stride + + s_offs[:, None] * dq_s_stride + + v_head * dq_h_stride + + chan_off[None, :] + ) + dk_ptrs = ( + dk_ptr + + batch_id * dk_b_stride + + s_offs[:, None] * dk_s_stride + + v_head * dk_h_stride + + chan_off[None, :] + ) + d_normed += tl.load( + dq_ptrs, mask=s_mask[:, None] & is_query, other=0.0 + ).to(tl.float32) + d_normed += tl.load( + dk_ptrs, mask=s_mask[:, None] & is_key, other=0.0 + ).to(tl.float32) + + silu_ptrs = ( + silu_bf16_ptr + + batch_id * silu_b_stride + + chan[None, :] * silu_c_stride + + s_offs[:, None] * silu_s_stride + ) + silu_bf16 = tl.load(silu_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + + norm_sq = tl.sum(silu_bf16 * silu_bf16, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + s_row = tl.sum(d_normed * silu_bf16, axis=1) + rstd3 = rstd * rstd * rstd + d_silu = rstd[:, None] * d_normed - rstd3[:, None] * silu_bf16 * s_row[:, None] + + d_silu_ptrs = ( + d_silu_bf16_ptr + + batch_id * d_silu_b_stride + + chan[None, :] * d_silu_c_stride + + s_offs[:, None] * d_silu_s_stride + ) + tl.store(d_silu_ptrs, d_silu.to(d_silu_bf16_ptr.dtype.element_ty), mask=s_mask[:, None]) + + +@triton.jit +def _v_layout_to_conv_kernel( + dv_ptr, # (b, s, num_v_heads, value_head_dim) + d_silu_conv_ptr, # (b, conv_dim, s) — write into V channel slice + seq_len, + num_v_heads, + v_channel_offset, # = 2 * qk_channels + dv_b_stride, + dv_s_stride, + dv_h_stride, + d_silu_b_stride, + d_silu_c_stride, + d_silu_s_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Write V-branch gradients into the conv-backward layout. + + ``dv`` is the gradient of ``value`` (forward layout + ``(b, s, num_v_heads, value_head_dim)``). The conv backward needs + ``d_silu_conv`` in layout ``(b, conv_dim, s)`` for the V channel + slice. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # Read dv at (batch, s, head, chan). + dv_ptrs = ( + dv_ptr + + batch_id * dv_b_stride + + s_offs[:, None] * dv_s_stride + + head_id * dv_h_stride + + chan_off[None, :] + ) + dv_val = tl.load(dv_ptrs, mask=s_mask[:, None], other=0.0) + + # Write to d_silu_conv at (batch, v_channel_offset + head*HEAD_DIM + chan, s). + d_silu_chan = v_channel_offset + head_id * HEAD_DIM + chan_off + d_silu_ptrs = ( + d_silu_conv_ptr + + batch_id * d_silu_b_stride + + d_silu_chan[None, :] * d_silu_c_stride + + s_offs[:, None] * d_silu_s_stride + ) + tl.store(d_silu_ptrs, dv_val, mask=s_mask[:, None]) + + +@triton.jit +def _z_layout_to_qkvzba_kernel( + dgate_ptr, # (b, s, num_v_heads, value_head_dim) + d_qkvzba_ptr, # (s, b, total_channels) — write into z channel slice + seq_len, + num_v_heads, + z_channel_offset, # = 2 * qk_channels + v_channels + dgate_b_stride, + dgate_s_stride, + dgate_h_stride, + d_qkvzba_s_stride, + d_qkvzba_b_stride, + d_qkvzba_c_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Write gate gradients into the z slice of ``d_qkvzba``. + + ``dgate`` is the autograd-supplied gradient of ``gate`` (= the z + slice of qkvzba in forward) with layout + ``(b, s, num_v_heads, value_head_dim)``. We need to write it into + ``d_qkvzba``'s z slice — layout ``(s, b, total_channels)`` with + channels in ``[z_channel_offset, z_channel_offset + v_channels)``. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # Read dgate at (batch, s, head, chan). + dgate_ptrs = ( + dgate_ptr + + batch_id * dgate_b_stride + + s_offs[:, None] * dgate_s_stride + + head_id * dgate_h_stride + + chan_off[None, :] + ) + dgate_val = tl.load(dgate_ptrs, mask=s_mask[:, None], other=0.0) + + # Write to d_qkvzba at (s, batch, z_channel_offset + head*HEAD_DIM + chan). + d_qkvzba_chan = z_channel_offset + head_id * HEAD_DIM + chan_off + d_qkvzba_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * d_qkvzba_s_stride + + batch_id * d_qkvzba_b_stride + + d_qkvzba_chan[None, :] * d_qkvzba_c_stride + ) + tl.store(d_qkvzba_ptrs, dgate_val, mask=s_mask[:, None]) + + +@triton.autotune( + configs=_g_beta_autotune_configs(), + key=["seq_len", "num_v_heads"], + # Each autotune trial atomic-adds partial sums into these accumulators. + # Without reset_to_zero the trials would stack on top of one another and + # produce values that are ``num_trials`` × the correct result. + reset_to_zero=["d_A_log_ptr", "d_dt_bias_ptr"], +) +@triton.jit +def _g_beta_backward_kernel( + qkvzba_ptr, + A_log_ptr, + dt_bias_ptr, + d_g_ptr, + d_beta_out_ptr, + d_qkvzba_ptr, + d_A_log_ptr, + d_dt_bias_ptr, + seq_len, + num_v_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + d_g_b_stride, + d_g_s_stride, + d_g_h_stride, + d_beta_b_stride, + d_beta_s_stride, + d_beta_h_stride, + BLOCK_S: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """Backward for ``_compute_g_and_beta_kernel``. + + Forward: + pre = alpha + dt_bias # fp32 + softplus_pre = log(1 + exp(pre)) + g = -exp(A_log) * softplus_pre + beta_sig = sigmoid(beta_raw) + + Backward (given d_g and d_beta_out): + d_alpha = d_g * (-exp(A_log) * sigmoid(pre)) + d_beta_raw = d_beta_out * beta_sig * (1 - beta_sig) + d_dt_bias[h] = Σ_{b,s} d_alpha[b,s,h] + d_A_log[h] = Σ_{b,s} d_g[b,s,h] * g[b,s,h] + + ``d_alpha`` and ``d_beta_raw`` are written into the matching channel slices + of ``d_qkvzba``. ``d_A_log`` and ``d_dt_bias`` are reduced via per-element + atomic_add to fp32 buffers; the caller casts those to the parameter dtype. + """ + + pid_b = tl.program_id(0) + pid_s = tl.program_id(1) + pid_h = tl.program_id(2) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + s_mask = s_offs < seq_len + h_mask = h_offs < num_v_heads + mask = s_mask[:, None] & h_mask[None, :] + + # ----- Forward recompute ----- + alpha_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + beta_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + alpha = tl.load(alpha_ptr, mask=mask, other=0.0).to(tl.float32) + beta_raw = tl.load(beta_ptr, mask=mask, other=0.0).to(tl.float32) + A_log = tl.load(A_log_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + + pre = alpha + dt_bias[None, :] + sigmoid_pre = tl.sigmoid(pre) + softplus_pre = tl.log(1.0 + tl.exp(pre)) + exp_A = tl.exp(A_log)[None, :] + g = -exp_A * softplus_pre + beta_sig = tl.sigmoid(beta_raw) + + # ----- Load upstream gradients ----- + d_g_ptrs = ( + d_g_ptr + + pid_b * d_g_b_stride + + s_offs[:, None] * d_g_s_stride + + h_offs[None, :] * d_g_h_stride + ) + d_beta_out_ptrs = ( + d_beta_out_ptr + + pid_b * d_beta_b_stride + + s_offs[:, None] * d_beta_s_stride + + h_offs[None, :] * d_beta_h_stride + ) + d_g = tl.load(d_g_ptrs, mask=mask, other=0.0).to(tl.float32) + d_beta_out = tl.load(d_beta_out_ptrs, mask=mask, other=0.0).to(tl.float32) + + # ----- Per-element gradients ----- + d_alpha = d_g * (-exp_A * sigmoid_pre) + d_beta_raw = d_beta_out * beta_sig * (1.0 - beta_sig) + + # ----- (b, s) → h reductions ----- + d_g_masked = tl.where(mask, d_g, 0.0) + d_alpha_masked = tl.where(mask, d_alpha, 0.0) + d_A_log_partial = tl.sum(d_g_masked * g, axis=0) + d_dt_bias_partial = tl.sum(d_alpha_masked, axis=0) + + # ----- Store per-element grads back to d_qkvzba ----- + d_alpha_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + d_beta_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + tl.store( + d_alpha_ptrs, d_alpha.to(d_qkvzba_ptr.dtype.element_ty), mask=mask + ) + tl.store( + d_beta_ptrs, d_beta_raw.to(d_qkvzba_ptr.dtype.element_ty), mask=mask + ) + + # ----- Atomic-add (b, s) partials into per-head accumulators ----- + tl.atomic_add(d_A_log_ptr + h_offs, d_A_log_partial, mask=h_mask) + tl.atomic_add(d_dt_bias_ptr + h_offs, d_dt_bias_partial, mask=h_mask) + + +# --------------------------------------------------------------------------- +# Python entry points +# --------------------------------------------------------------------------- + + + + +def _is_power_of_two(value: int) -> bool: + return value > 0 and (value & (value - 1)) == 0 + + +_SIDE_STREAMS: dict = {} + + +def _get_side_stream(device: torch.device, slot: int) -> "torch.cuda.Stream": + """Lazily allocate and cache CUDA streams keyed by ``(device, slot)``. + + Reusing streams across calls keeps launches free of stream-creation + overhead, which would otherwise dominate the small kernels. + """ + + key = (device.index if device.index is not None else torch.cuda.current_device(), slot) + stream = _SIDE_STREAMS.get(key) + if stream is None: + stream = torch.cuda.Stream(device=device) + _SIDE_STREAMS[key] = stream + return stream + + +def _triton_l2norm_repeat_backward( + d_qk_out: Tensor, + silu_bf16: Tensor, + d_silu_bf16: Tensor, + *, + is_query: bool, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + eps: float = 1e-6, + stream: Optional["torch.cuda.Stream"] = None, +) -> Tensor: + """l2norm + REPEAT backward. + + ``silu_bf16`` is the (b, conv_dim, s) bf16 tensor produced by re-running + causal_conv1d_fn (forward, no-grad). Output ``d_silu_bf16`` is written + in place; only the matching channel slice (Q or K) is filled in. + """ + + batch = d_qk_out.shape[0] + seq_len = d_qk_out.shape[1] + qk_channels = num_key_heads * key_head_dim + repeat = num_value_heads // num_key_heads + channel_offset = 0 if is_query else qk_channels + + device = d_qk_out.device + + grid = lambda meta: ( + batch * num_key_heads, + triton.cdiv(seq_len, meta["BLOCK_S"]), + ) + + with _launch_context(device, stream): + _l2norm_repeat_backward_kernel[grid]( + d_qk_out, + silu_bf16, + d_silu_bf16, + seq_len, + num_key_heads, + channel_offset, + eps, + d_qk_out.stride(0), + d_qk_out.stride(1), + d_qk_out.stride(2), + silu_bf16.stride(0), + silu_bf16.stride(1), + silu_bf16.stride(2), + d_silu_bf16.stride(0), + d_silu_bf16.stride(1), + d_silu_bf16.stride(2), + HEAD_DIM=key_head_dim, + REPEAT=repeat, + ) + + return d_silu_bf16 + + +def _triton_qk_l2norm_repeat_backward( + dq: Tensor, + dk: Tensor, + silu_bf16: Tensor, + d_silu_bf16: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + eps: float = 1e-6, + stream: Optional["torch.cuda.Stream"] = None, +) -> Tensor: + """Merged Q/K l2norm + REPEAT backward launch.""" + + batch = dq.shape[0] + seq_len = dq.shape[1] + qk_channels = num_key_heads * key_head_dim + repeat = num_value_heads // num_key_heads + device = dq.device + + grid = lambda meta: ( + batch * 2 * num_key_heads, + triton.cdiv(seq_len, meta["BLOCK_S"]), + ) + + with _launch_context(device, stream): + _qk_l2norm_repeat_backward_kernel[grid]( + dq, + dk, + silu_bf16, + d_silu_bf16, + seq_len, + num_key_heads, + qk_channels, + eps, + dq.stride(0), + dq.stride(1), + dq.stride(2), + dk.stride(0), + dk.stride(1), + dk.stride(2), + silu_bf16.stride(0), + silu_bf16.stride(1), + silu_bf16.stride(2), + d_silu_bf16.stride(0), + d_silu_bf16.stride(1), + d_silu_bf16.stride(2), + HEAD_DIM=key_head_dim, + REPEAT=repeat, + ) + + return d_silu_bf16 + + +def _triton_v_layout_to_conv( + dv: Tensor, + d_silu_conv: Tensor, + *, + v_channel_offset: int, + num_value_heads: int, + value_head_dim: int, + stream: Optional["torch.cuda.Stream"] = None, +) -> None: + """Write ``dv`` into ``d_silu_conv``'s V channel slice.""" + + batch, seq_len, _, _ = dv.shape + device = dv.device + + BLOCK_S = _LAYOUT_BLOCK_S + num_seq_blocks = triton.cdiv(seq_len, BLOCK_S) + grid = (batch * num_value_heads, num_seq_blocks) + + with _launch_context(device, stream): + _v_layout_to_conv_kernel[grid]( + dv, + d_silu_conv, + seq_len, + num_value_heads, + v_channel_offset, + dv.stride(0), + dv.stride(1), + dv.stride(2), + d_silu_conv.stride(0), + d_silu_conv.stride(1), + d_silu_conv.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_S, + num_warps=4, + num_stages=2, + ) + + +def _triton_z_layout_to_qkvzba( + dgate: Tensor, + d_qkvzba: Tensor, + *, + z_channel_offset: int, + num_value_heads: int, + value_head_dim: int, + stream: Optional["torch.cuda.Stream"] = None, +) -> None: + """Write ``dgate`` into ``d_qkvzba``'s z channel slice.""" + + batch, seq_len, _, _ = dgate.shape + device = dgate.device + + BLOCK_S = _LAYOUT_BLOCK_S + num_seq_blocks = triton.cdiv(seq_len, BLOCK_S) + grid = (batch * num_value_heads, num_seq_blocks) + + with _launch_context(device, stream): + _z_layout_to_qkvzba_kernel[grid]( + dgate, + d_qkvzba, + seq_len, + num_value_heads, + z_channel_offset, + dgate.stride(0), + dgate.stride(1), + dgate.stride(2), + d_qkvzba.stride(0), + d_qkvzba.stride(1), + d_qkvzba.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_S, + num_warps=4, + num_stages=2, + ) + + +def _triton_g_beta_backward( + qkvzba: Tensor, + A_log: Tensor, + dt_bias: Tensor, + d_g: Tensor, + d_beta_out: Tensor, + *, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + num_key_heads: int, + d_qkvzba_out: Optional[Tensor] = None, + stream: Optional["torch.cuda.Stream"] = None, +) -> Tuple[Tensor, Tensor, Tensor]: + """Launch ``_g_beta_backward_kernel`` and return its outputs. + + Returns: + ``(d_qkvzba_out, d_A_log, d_dt_bias)``. ``d_qkvzba_out`` only has its + alpha and beta slices filled in; the caller is expected to allocate + the buffer while the other backward kernels fill the rest. + ``d_A_log`` and ``d_dt_bias`` are fp32 and need to be cast back to + the parameter dtype by the caller. + """ + + seq_len, batch, total_channels = qkvzba.shape + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + beta_channel_offset = 2 * qk_channels + 2 * v_channels + alpha_channel_offset = beta_channel_offset + num_value_heads + + if d_qkvzba_out is None: + d_qkvzba_out = torch.zeros_like(qkvzba) + + device = qkvzba.device + + g_beta_grid = lambda meta: ( + batch, + triton.cdiv(seq_len, meta["BLOCK_S"]), + triton.cdiv(num_value_heads, meta["BLOCK_H"]), + ) + with _launch_context(device, stream): + d_param_grads = torch.empty((2, num_value_heads), dtype=torch.float32, device=device) + d_param_grads.zero_() + d_A_log = d_param_grads[0] + d_dt_bias = d_param_grads[1] + _g_beta_backward_kernel[g_beta_grid]( + qkvzba, + A_log, + dt_bias, + d_g, + d_beta_out, + d_qkvzba_out, + d_A_log, + d_dt_bias, + seq_len, + num_value_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + d_g.stride(0), + d_g.stride(1), + d_g.stride(2), + d_beta_out.stride(0), + d_beta_out.stride(1), + d_beta_out.stride(2), + ) + return d_qkvzba_out, d_A_log, d_dt_bias + + +class _NullContext: + def __enter__(self): + return None + + def __exit__(self, exc_type, exc_val, exc_tb): + return False + + +def _launch_context( + device: torch.device, + stream: Optional["torch.cuda.Stream"], +): + """Return a CUDA launch context after wiring the optional side stream.""" + + if stream is None: + return _NullContext() + stream.wait_stream(torch.cuda.current_stream(device)) + return torch.cuda.stream(stream) + + +def _wait_for_streams( + dst_stream: "torch.cuda.Stream", + *src_streams: "torch.cuda.Stream", +) -> None: + for stream in src_streams: + dst_stream.wait_stream(stream) + + +def _resolve_packed_seq_idx( + cu_seqlens: Optional[Tensor], + seq_idx: Optional[Tensor], + total_tokens: int, +) -> Optional[Tensor]: + """Return the token-level sequence-id buffer for causal-conv backward.""" + + if cu_seqlens is None: + assert seq_idx is None, "seq_idx requires cu_seqlens for packed THD mode." + return None + + if seq_idx is None: + seq_lengths = cu_seqlens[1:] - cu_seqlens[:-1] + seq_idx = torch.repeat_interleave( + torch.arange(seq_lengths.numel(), device=cu_seqlens.device, dtype=torch.int32), + seq_lengths, + ) + seq_idx = seq_idx.unsqueeze(0) + elif seq_idx.dim() == 1: + seq_idx = seq_idx.unsqueeze(0) + + assert seq_idx.is_cuda, f"Packed seq_idx must be CUDA, got {seq_idx.device}." + assert seq_idx.dtype == torch.int32, f"Packed seq_idx must be int32, got {seq_idx.dtype}." + assert seq_idx.shape == (1, total_tokens), ( + "Packed seq_idx must have shape [1, total_tokens], " + f"got {seq_idx.shape=} and {total_tokens=}." + ) + return seq_idx.contiguous() + + +def _triton_pre_gated_delta_rule_forward( + qkvzba: Tensor, + conv1d_weight: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + cu_seqlens: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Triton-backed forward for the pre-gated-delta-rule front-end. + + Returns ``(query, key, value, gate, beta, g, silu_qk_save)``. The last + element is the bf16-rounded ``silu(conv(x))`` for the QK channel range + laid out channel-last so the backward can feed it straight into + ``causal_conv1d_bwd_function`` — see module docstring. + """ + + seq_len, batch, total_channels = qkvzba.shape + is_packed_thd = cu_seqlens is not None + if is_packed_thd: + assert batch == 1, ( + "Packed THD fused_pre_gated_delta_rule expects batch dimension 1; " + f"got {batch=}." + ) + num_packed_seqs = cu_seqlens.shape[0] - 1 + else: + num_packed_seqs = 0 + + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + repeat_factor = num_value_heads // num_key_heads + k_w = conv1d_weight.shape[-1] + assert _is_power_of_two(key_head_dim), ( + "Triton kernel currently expects key_head_dim to be a power of two; " + f"got {key_head_dim=}." + ) + assert _is_power_of_two(value_head_dim), ( + "Triton kernel currently expects value_head_dim to be a power of two; " + f"got {value_head_dim=}." + ) + + expected_channels = 2 * qk_channels + 2 * v_channels + 2 * num_value_heads + assert total_channels == expected_channels, ( + f"qkvzba last-dim mismatch: got {total_channels}, expected {expected_channels}." + ) + + out_dtype = qkvzba.dtype + device = qkvzba.device + + # Output buffers: contiguous (b, s, h, d) for q/k/v and (b, s, h) for g/beta. + # Q and K share one allocation so the fused-streamed QK kernel can select + # the logical group by pointer stride instead of branching between two + # unrelated base pointers inside Triton. + qk_out = torch.empty( + 2, batch, seq_len, num_value_heads, key_head_dim, dtype=out_dtype, device=device + ) + query = qk_out[0] + key = qk_out[1] + value = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + g = torch.empty(batch, seq_len, num_value_heads, dtype=torch.float32, device=device) + beta = torch.empty(batch, seq_len, num_value_heads, dtype=out_dtype, device=device) + + # Conv weight is (conv_dim, 1, K_W); we treat it as (conv_dim, K_W). + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + + # No conv bias support: the entry point asserts this. We still pass a + # dummy ``bias_tensor`` to the kernel so the launch signature stays + # stable; ``HAS_BIAS=False`` ensures the kernel never reads it. + bias_tensor = qkvzba + bias_stride = 0 + + # Allocate the gate (z) output buffer that the independent Z kernel will + # populate. Keeping Z separate makes the forward scopes QK / V / Z / + # G-Beta explicit. + gate = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + + # Persist the QK silu(conv(x)) intermediate in channel-last layout so the + # backward can feed it directly into the l2norm backward. + silu_qk_save = torch.empty( + (batch, seq_len, 2 * qk_channels), dtype=out_dtype, device=device + ).permute(0, 2, 1) # → (b, 2*qk_c, s) with stride(1)==1 + silu_save_b_stride = silu_qk_save.stride(0) + silu_save_c_stride = silu_qk_save.stride(1) + silu_save_s_stride = silu_qk_save.stride(2) + + # Stream setup. Each side stream handles one of the four sub-computations + # (QK conv+l2norm, V conv, Z copy, g/beta). + main_stream = torch.cuda.current_stream(device=device) + qk_stream = _get_side_stream(device, slot=_QK_STREAM_SLOT) + v_stream = _get_side_stream(device, slot=_V_STREAM_SLOT) + g_beta_stream = _get_side_stream(device, slot=_G_BETA_STREAM_SLOT) + z_stream = _get_side_stream(device, slot=_Z_STREAM_SLOT) + for stream in (qk_stream, v_stream, g_beta_stream, z_stream): + stream.wait_stream(main_stream) + + # --- QK conv + silu + l2norm + repeat --- + qk_grid = lambda meta: ( + batch * 2 * num_key_heads, + triton.cdiv(seq_len, meta["BLOCK_S"]), + ) + with torch.cuda.stream(qk_stream): + if is_packed_thd: + _conv_silu_project_thd_kernel[qk_grid]( + qkvzba, + weight_2d, + bias_tensor, + qk_out, + silu_qk_save, + cu_seqlens, + seq_len, + num_packed_seqs, + num_key_heads, + 0, # QK starts at channel 0; group 1 starts at +qk_channels. + qk_channels, + 0, # silu_save_chan_offset + qk_channels, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + qk_out.stride(0), + qk_out.stride(1), + qk_out.stride(2), + qk_out.stride(3), + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + _L2NORM_EPS, + HEAD_DIM=key_head_dim, + K_W=k_w, + REPEAT=repeat_factor, + NUM_GROUPS=2, + HAS_BIAS=False, + SAVE_SILU=True, + APPLY_L2=True, + ) + else: + _conv_silu_project_kernel[qk_grid]( + qkvzba, + weight_2d, + bias_tensor, + qk_out, + silu_qk_save, + seq_len, + num_key_heads, + 0, # QK starts at channel 0; group 1 starts at +qk_channels. + qk_channels, + 0, # silu_save_chan_offset + qk_channels, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + qk_out.stride(0), + qk_out.stride(1), + qk_out.stride(2), + qk_out.stride(3), + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + _L2NORM_EPS, + HEAD_DIM=key_head_dim, + K_W=k_w, + REPEAT=repeat_factor, + NUM_GROUPS=2, + HAS_BIAS=False, + SAVE_SILU=True, + APPLY_L2=True, + ) + + # --- V conv + silu (no l2norm, no repeat) --- + v_channel_offset = 2 * qk_channels + z_channel_offset = 2 * qk_channels + v_channels + v_grid = lambda meta: (batch * num_value_heads, triton.cdiv(seq_len, meta["BLOCK_S"])) + with torch.cuda.stream(v_stream): + if is_packed_thd: + _conv_silu_project_thd_kernel[v_grid]( + qkvzba, + weight_2d, + bias_tensor, + value, + qkvzba, # silu_save unused (SAVE_SILU=False) + cu_seqlens, + seq_len, + num_packed_seqs, + num_value_heads, + v_channel_offset, + 0, # in_group_stride unused for NUM_GROUPS=1 + 0, # silu_save_chan_offset unused + 0, # silu_save_group_stride unused + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + 0, # out_group_dim_stride unused for NUM_GROUPS=1 + value.stride(0), + value.stride(1), + value.stride(2), + 0, # silu_save strides unused + 0, + 0, + _L2NORM_EPS, + HEAD_DIM=value_head_dim, + K_W=k_w, + REPEAT=1, + NUM_GROUPS=1, + HAS_BIAS=False, + SAVE_SILU=False, + APPLY_L2=False, + ) + else: + _conv_silu_project_kernel[v_grid]( + qkvzba, + weight_2d, + bias_tensor, + value, + qkvzba, # silu_save unused (SAVE_SILU=False) + seq_len, + num_value_heads, + v_channel_offset, + 0, # in_group_stride unused for NUM_GROUPS=1 + 0, # silu_save_chan_offset unused + 0, # silu_save_group_stride unused + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + 0, # out_group_dim_stride unused for NUM_GROUPS=1 + value.stride(0), + value.stride(1), + value.stride(2), + 0, # silu_save strides unused + 0, + 0, + _L2NORM_EPS, + HEAD_DIM=value_head_dim, + K_W=k_w, + REPEAT=1, + NUM_GROUPS=1, + HAS_BIAS=False, + SAVE_SILU=False, + APPLY_L2=False, + ) + + # --- Z copy --- + BLOCK_Z_S = _LAYOUT_BLOCK_S + z_grid = (batch * num_value_heads, triton.cdiv(seq_len, BLOCK_Z_S)) + with torch.cuda.stream(z_stream): + _copy_z_kernel[z_grid]( + qkvzba, + gate, + seq_len, + num_value_heads, + z_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + gate.stride(0), + gate.stride(1), + gate.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_Z_S, + num_warps=4, + num_stages=2, + ) + + # --- g and beta --- + beta_channel_offset = 2 * qk_channels + 2 * v_channels + alpha_channel_offset = beta_channel_offset + num_value_heads + g_beta_grid = lambda meta: ( + batch, + triton.cdiv(seq_len, meta["BLOCK_S"]), + triton.cdiv(num_value_heads, meta["BLOCK_H"]), + ) + with torch.cuda.stream(g_beta_stream): + _compute_g_and_beta_kernel[g_beta_grid]( + qkvzba, + A_log, + dt_bias, + g, + beta, + seq_len, + num_value_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + g.stride(0), + g.stride(1), + g.stride(2), + beta.stride(0), + beta.stride(1), + beta.stride(2), + ) + + # Re-join the side streams so the caller's stream observes the writes. + _wait_for_streams(main_stream, qk_stream, v_stream, z_stream, g_beta_stream) + + return query, key, value, gate, beta, g, silu_qk_save + + +def _triton_pre_gated_delta_rule_backward( + qkvzba: Tensor, + conv1d_weight: Tensor, + silu_qk_save: Tensor, + dq: Tensor, + dk: Tensor, + dv: Tensor, + dgate: Tensor, + dbeta: Tensor, + dg: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Triton-backed backward for the pre-gated-delta-rule front-end. + + Mirror of :func:`_triton_pre_gated_delta_rule_forward`. Takes upstream + gradients (``dq``/``dk``/``dv``/``dgate``/``dbeta``/``dg``) plus the + saved forward intermediates and returns input/parameter gradients + ``(d_qkvzba, d_weight, d_A_log, d_dt_bias)``. + + Five Triton kernels + one C++ ``causal_conv1d_bwd_function`` call, + fanned out on five side streams so memory-bound work overlaps while + the conv backward runs on the default stream. See module docstring + for the overall design. + """ + + seq_len, batch, _ = qkvzba.shape + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + conv_dim = 2 * qk_channels + v_channels + z_offset = 2 * qk_channels + v_channels + k_w = conv1d_weight.shape[-1] + device = qkvzba.device + + # Rebuild the conv input as a NON-contiguous (b, c, s) view of qkvzba. + # ``causal_conv1d_fn`` / ``_bwd_function`` accept inputs where either + # ``stride(1) == 1`` or ``stride(2) == 1``; the permuted view of qkvzba + # satisfies the former (channel stride is 1 in the original (s, b, c) + # layout), so we can skip a 256 MB ``.contiguous()`` copy. + qkvzba_conv = qkvzba[:, :, :conv_dim].permute(1, 2, 0) + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + + # ``silu_qk_save`` is the (b, 2*qk_channels, s) bf16 buffer the + # forward wrote ``silu(conv(x))`` into for QK. Reuse it directly as + # the silu input to the l2norm backward. + silu_conv = silu_qk_save + + # Allocate d_silu_conv channel-last (stride(1)==1) — that's what + # ``causal_conv1d_channellast_bwd_kernel`` consumes natively. + d_silu_conv = torch.empty( + (batch, seq_len, conv_dim), dtype=qkvzba.dtype, device=device + ).permute(0, 2, 1) + + # Use the same stream slots as the forward for the matching scopes. + qk_stream = _get_side_stream(device, slot=_QK_STREAM_SLOT) + v_stream = _get_side_stream(device, slot=_V_STREAM_SLOT) + g_beta_stream = _get_side_stream(device, slot=_G_BETA_STREAM_SLOT) + z_stream = _get_side_stream(device, slot=_Z_STREAM_SLOT) + + # Q + K: l2norm + REPEAT backward writes into d_silu_conv's Q/K slices. + _triton_qk_l2norm_repeat_backward( + dq, + dk, + silu_conv, + d_silu_conv, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + stream=qk_stream, + ) + + # V: no l2norm and no REPEAT in forward, so d_silu_conv's V slice is + # just dv re-laid-out from (b, s, num_v_heads, value_head_dim) to + # (b, v_channels, s). + _triton_v_layout_to_conv( + dv, + d_silu_conv, + v_channel_offset=2 * qk_channels, + num_value_heads=num_value_heads, + value_head_dim=value_head_dim, + stream=v_stream, + ) + + # g + beta backward fully stores d_qkvzba's alpha + beta slices, plus + # per-head d_A_log / d_dt_bias. Conv and z slices are filled by the + # causal-conv and z kernels, so d_qkvzba does not need a pre-zero. + d_qkvzba = torch.empty_like(qkvzba) + _, d_A_log_fp32, d_dt_bias_fp32 = _triton_g_beta_backward( + qkvzba, + A_log, + dt_bias, + dg, + dbeta, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + num_key_heads=num_key_heads, + d_qkvzba_out=d_qkvzba, + stream=g_beta_stream, + ) + + # Z slice gradient: stream dgate into d_qkvzba's z slice. + _triton_z_layout_to_qkvzba( + dgate, + d_qkvzba, + z_channel_offset=z_offset, + num_value_heads=num_value_heads, + value_head_dim=value_head_dim, + stream=z_stream, + ) + + # Join only streams that wrote into d_silu_conv before causal_conv1d_bwd_function. + # g/beta and z write disjoint outputs and can continue overlapping with conv bwd. + default_stream = torch.cuda.current_stream(device) + _wait_for_streams(default_stream, qk_stream, v_stream) + + # Pre-allocate d_x_conv as a strided view INTO d_qkvzba's conv slice. + # d_qkvzba memory layout is (s, b, total_channels) contiguous, so + # element [s, b, c] sits at offset s*b_stride + b*c_stride + c. + # Re-interpreting that storage as (b, conv_dim, s) lets + # causal_conv1d_bwd_function write d_x directly into the right cells. + seq_stride = qkvzba.stride(0) + batch_stride = qkvzba.stride(1) + d_x_conv_view = d_qkvzba.as_strided( + (batch, conv_dim, seq_len), + (batch_stride, 1, seq_stride), + ) + + # Hand-tuned C++ conv backward. Internally folds the silu' factor and + # computes both d_x and d_w in fp32; writes d_x directly into the + # view above. + _, d_weight_fp32, _, _ = _causal_conv1d_bwd_function( + qkvzba_conv, + weight_2d, + None, # no bias + d_silu_conv, + seq_idx, + None, # initial_states + None, # dfinal_states + d_x_conv_view, # dx pre-allocated into d_qkvzba's conv slice + False, # return_dinitial_states + True, # activation (silu) + ) + + d_weight = d_weight_fp32.view(*conv1d_weight.shape).to(conv1d_weight.dtype) + default_stream.wait_stream(g_beta_stream) + d_A_log = d_A_log_fp32.to(A_log.dtype) + d_dt_bias = d_dt_bias_fp32.to(dt_bias.dtype) + default_stream.wait_stream(z_stream) + + return d_qkvzba, d_weight, d_A_log, d_dt_bias + + +class _FusedPreGatedDeltaRuleFunction(torch.autograd.Function): + """Thin :class:`torch.autograd.Function` wrapper around the fused path. + + Stashes the forward inputs + the saved ``silu_qk_save`` intermediate + in ``ctx`` and dispatches to :func:`_triton_pre_gated_delta_rule_forward` + / :func:`_triton_pre_gated_delta_rule_backward`. The actual kernel + logic lives in those two free functions so it's easy to read and + reuse outside the autograd machinery. + """ + + @staticmethod + def forward( + ctx, + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ): + ctx.num_key_heads = num_key_heads + ctx.num_value_heads = num_value_heads + ctx.key_head_dim = key_head_dim + ctx.value_head_dim = value_head_dim + query, key, value, gate, beta, g, silu_qk_save = ( + _triton_pre_gated_delta_rule_forward( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + cu_seqlens=cu_seqlens, + ) + ) + ctx.has_seq_idx = seq_idx is not None + if ctx.has_seq_idx: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx) + else: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save) + return query, key, value, gate, beta, g + + @staticmethod + def backward(ctx, dq, dk, dv, dgate, dbeta, dg): + if ctx.has_seq_idx: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx = ctx.saved_tensors + else: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save = ctx.saved_tensors + seq_idx = None + d_qkvzba, d_weight, d_A_log, d_dt_bias = _triton_pre_gated_delta_rule_backward( + qkvzba, + conv1d_weight, + silu_qk_save, + dq, + dk, + dv, + dgate, + dbeta, + dg, + A_log, + dt_bias, + num_key_heads=ctx.num_key_heads, + num_value_heads=ctx.num_value_heads, + key_head_dim=ctx.key_head_dim, + value_head_dim=ctx.value_head_dim, + seq_idx=seq_idx, + ) + # Match forward inputs: (qkvzba, conv1d_weight, A_log, dt_bias, + # cu_seqlens, seq_idx, num_key_heads, num_value_heads, + # key_head_dim, value_head_dim). + # Non-tensor args get None. + return ( + d_qkvzba, + d_weight, + d_A_log, + d_dt_bias, + None, + None, + None, + None, + None, + None, + ) + + +def fused_streamed_pre_gated_delta_rule( + qkvzba: Tensor, + conv1d_weight: Tensor, + conv1d_bias: Optional[Tensor], + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + use_qk_l2norm: bool = True, + cu_seqlens: Optional[Tensor] = None, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Streamed fused pre-gated-delta-rule entry point. + + Args: + qkvzba: ``[seq_len, batch, in_proj_dim]`` projection output. Must be + on CUDA. + conv1d_weight: ``[conv_dim, 1, k_w]`` depthwise conv weight. + conv1d_bias: Must be ``None`` (conv bias is not supported). + A_log: ``[num_value_heads]`` raw decay parameter. + dt_bias: ``[num_value_heads]`` time-step bias. + num_key_heads / num_value_heads / key_head_dim / value_head_dim: GDN + architecture parameters. ``num_value_heads`` must be an integer + multiple of ``num_key_heads``. + use_qk_l2norm: Must be ``True``; the fused backward closes over the + l2norm path. + cu_seqlens: Optional packed THD cumulative sequence lengths. When set, + ``qkvzba`` must have ``batch == 1`` and ``cu_seqlens[-1] == seq_len``. + seq_idx: Optional precomputed token-to-sequence map with shape + ``[1, seq_len]``. Used by causal-conv backward in packed THD mode. + + Returns: + ``(query, key, value, gate, beta, g)`` matching the unfused + :meth:`GatedDeltaNet.pre_gated_delta_rule` API. + """ + + assert qkvzba.is_cuda, ( + "fused_pre_gated_delta_rule requires CUDA inputs; " + f"got qkvzba.device={qkvzba.device}." + ) + assert conv1d_bias is None, ( + "Conv bias is not supported by fused_pre_gated_delta_rule " + "(production GDN config has none)." + ) + assert use_qk_l2norm, ( + "use_qk_l2norm=False is not supported by fused_pre_gated_delta_rule " + "(the backward closes over the l2norm path)." + ) + assert num_value_heads % num_key_heads == 0, ( + f"{num_value_heads=} must be a multiple of {num_key_heads=}." + ) + if cu_seqlens is not None: + assert cu_seqlens.is_cuda, ( + "Packed fused_pre_gated_delta_rule requires CUDA cu_seqlens; " + f"got cu_seqlens.device={cu_seqlens.device}." + ) + assert cu_seqlens.dtype == torch.int32, ( + "Packed fused_pre_gated_delta_rule requires int32 cu_seqlens; " + f"got {cu_seqlens.dtype=}." + ) + assert cu_seqlens.dim() == 1, ( + "Packed fused_pre_gated_delta_rule expects 1-D cu_seqlens; " + f"got {cu_seqlens.shape=}." + ) + assert qkvzba.shape[1] == 1, ( + "Packed THD fused_pre_gated_delta_rule expects batch dimension 1; " + f"got qkvzba.shape={qkvzba.shape}." + ) + assert cu_seqlens.shape[0] >= 2, ( + "Packed fused_pre_gated_delta_rule requires at least one packed sequence; " + f"got {cu_seqlens.shape=}." + ) + assert cu_seqlens[0].item() == 0, ( + "Packed fused_pre_gated_delta_rule requires cu_seqlens[0] == 0, " + f"got {cu_seqlens[0].item()}." + ) + assert torch.all(cu_seqlens[1:] >= cu_seqlens[:-1]).item(), ( + "Packed fused_pre_gated_delta_rule requires monotonically non-decreasing " + f"cu_seqlens, got {cu_seqlens}." + ) + assert cu_seqlens[-1].item() == qkvzba.shape[0], ( + "Packed fused_pre_gated_delta_rule requires cu_seqlens[-1] to match " + f"seq_len, got {cu_seqlens[-1].item()} vs {qkvzba.shape[0]}." + ) + cu_seqlens = cu_seqlens.contiguous() + seq_idx = _resolve_packed_seq_idx(cu_seqlens, seq_idx, qkvzba.shape[0]) + else: + assert seq_idx is None, "seq_idx requires cu_seqlens for packed THD mode." + + return _FusedPreGatedDeltaRuleFunction.apply( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ) + + +fused_pre_gated_delta_rule = fused_streamed_pre_gated_delta_rule diff --git a/megatron/core/models/common/embeddings/rotary_pos_embedding.py b/megatron/core/models/common/embeddings/rotary_pos_embedding.py index 804bdb7c537..77eb94a34bf 100644 --- a/megatron/core/models/common/embeddings/rotary_pos_embedding.py +++ b/megatron/core/models/common/embeddings/rotary_pos_embedding.py @@ -205,6 +205,47 @@ def forward( return emb + def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group=None): + """Materialize cached cos/sin tensors for ``[seq_len, ..., dim]``.""" + self.max_seq_len_cached = seq_len + self.offset_cached = offset + self.dtype_cached = dtype + self.packed_seq_cached = packed_seq + + emb = self.forward(seq_len, offset, packed_seq=packed_seq, cp_group=cp_group) + self.register_buffer("cos_cached", emb.cos().to(dtype).contiguous(), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype).contiguous(), persistent=False) + + def get_cached_cos_sin( + self, + seq_len, + offset=0, + dtype=torch.get_default_dtype(), + packed_seq=False, + cp_group=None, + mscale=None, + ): + """Get cached cos and sin values. + + The cache is rebuilt on first use or whenever ``seq_len`` grows + beyond the cached length, or any of ``offset`` / ``dtype`` / + ``packed_seq`` changes from the previous call. + ``YarnRotaryEmbedding`` overrides this to also bake its + concentration factor into the cached cos/sin (controlled by + ``mscale``); for the base class without a concentration + factor the argument is accepted-and-ignored for API uniformity. + """ + del mscale # base class has no concentration factor + if ( + not hasattr(self, "max_seq_len_cached") + or seq_len > self.max_seq_len_cached + or offset != self.offset_cached + or dtype != self.dtype_cached + or packed_seq != self.packed_seq_cached + ): + self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group) + return (self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]) + def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs): state_dict.pop(f'{prefix}inv_freq', None) return super()._load_from_state_dict(state_dict, prefix, *args, **kwargs) diff --git a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py index bc5a9c5fa3f..cb8a03d0b2b 100644 --- a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py +++ b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py @@ -186,13 +186,18 @@ def forward( emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group) return emb, _mscale - def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group=None): + def _set_cos_sin_cache( + self, seq_len, offset, dtype, packed_seq=False, cp_group=None, mscale=None + ): self.max_seq_len_cached = seq_len self.offset_cached = offset self.dtype_cached = dtype self.packed_seq_cached = packed_seq + self.mscale_cached = mscale emb, _mscale = self.forward(seq_len, offset, packed_seq=packed_seq, cp_group=cp_group) + if mscale is not None: + _mscale = mscale self.register_buffer( "cos_cached", (emb.cos() * _mscale).to(dtype).contiguous(), persistent=False ) @@ -201,16 +206,34 @@ def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group= ) def get_cached_cos_sin( - self, seq_len, offset=0, dtype=torch.get_default_dtype(), packed_seq=False, cp_group=None + self, + seq_len, + offset=0, + dtype=torch.get_default_dtype(), + packed_seq=False, + cp_group=None, + mscale=None, ): - """Get cached cos and sin values.""" + """Get cached cos and sin values. + + Args: + mscale: when ``None`` (default), the cached cos/sin are + multiplied by yarn's internal concentration factor (the + normal long-context behaviour). When a float is supplied, + that value is used in place of the internal factor — e.g. + the DSv4 hybrid model passes ``mscale=1.0`` to enforce + its "pure rotation" contract and keep the fused / + unfused rope paths bit-equivalent. + """ if ( - seq_len > self.max_seq_len_cached + not hasattr(self, "max_seq_len_cached") + or seq_len > self.max_seq_len_cached or offset != self.offset_cached or dtype != self.dtype_cached or packed_seq != self.packed_seq_cached + or mscale != getattr(self, "mscale_cached", None) ): - self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group) + self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group, mscale) return (self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]) diff --git a/megatron/core/models/hybrid/hybrid_block.py b/megatron/core/models/hybrid/hybrid_block.py index 93cb56f5297..bfdeb6029f1 100644 --- a/megatron/core/models/hybrid/hybrid_block.py +++ b/megatron/core/models/hybrid/hybrid_block.py @@ -7,7 +7,7 @@ from contextlib import nullcontext from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch from torch import Tensor, nn @@ -22,12 +22,21 @@ from megatron.core.models.hybrid.hybrid_layer_allocation import Symbols as LayerSymbols from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import CheckpointManager from megatron.core.transformer import TransformerConfig +from megatron.core.transformer.hyper_connection import ( + HyperConnectionModule, + learned_output_contract, +) from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.transformer_layer import TransformerLayer -from megatron.core.transformer.utils import sharded_state_dict_default +from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, + make_sharded_tensors_for_checkpoint, + sharded_state_dict_default, +) from megatron.core.utils import WrappedTensor, deprecate_inference_params, make_viewless_tensor @@ -46,6 +55,165 @@ class HybridStackSubmodules: mtp_block_spec: Optional[ModuleSpec] = None +class HyperConnectionHybridLayer(MegatronModule): + """Layer-boundary mHC wrapper for HybridStack layers. + + Hybrid layers already own their local residual paths. For this initial + integration we treat each hybrid layer as a single function by aggregating + n streams to the layer input, running the existing layer, and feeding only + the layer delta back through mHC expansion. The expansion path intentionally + uses zero additional dropout because the wrapped hybrid layer has already + applied its local dropout/residual update before the delta is computed. + + Checkpoint compatibility: this is a *wrapper* (the inner layer is held as + `self.inner_layer`), so wrapped-layer state_dict keys are nested under + `inner_layer.` (e.g. `layers.0.inner_layer.input_layernorm.weight` instead + of `layers.0.input_layernorm.weight`). HybridStack checkpoints saved with + `enable_hyper_connections=False` cannot be loaded into a model with + `enable_hyper_connections=True` (and vice versa) without a key-mapping + migration. Note: this differs from `HyperConnectionTransformerLayer`, + which subclasses `TransformerLayer` and only adds new sibling fields, + keeping all base keys stable. + """ + + def __init__(self, config: TransformerConfig, layer: MegatronModule) -> None: + super().__init__(config=config) + self.inner_layer = layer + self.layer_number = layer.layer_number + self.hyper_connection = HyperConnectionModule(config=config, layer_number=self.layer_number) + if config.params_dtype is not None: + self.hyper_connection.to(dtype=config.params_dtype) + if hasattr(layer, 'tp_group'): + self.tp_group = layer.tp_group + + def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int]]]: + """Delegate Mamba inference state shape requests to the wrapped layer.""" + if not hasattr(self.inner_layer, 'mamba_state_shapes_per_request'): + return None + return self.inner_layer.mamba_state_shapes_per_request() + + def _call_inner_layer( + self, + hidden_states: Tensor, + attention_mask: Tensor, + inference_context: Optional[BaseInferenceContext], + rotary_pos_emb: Optional[Tensor], + sequence_len_offset: Optional[Tensor], + packed_seq_params: Optional[PackedSeqParams], + padding_mask: Optional[Tensor], + ) -> Tuple[Tensor, Optional[Tensor]]: + if isinstance(self.inner_layer, TransformerLayer): + output = self.inner_layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + rotary_pos_emb=rotary_pos_emb, + sequence_len_offset=sequence_len_offset, + packed_seq_params=packed_seq_params, + padding_mask=padding_mask, + _called_from_hybrid_mhc_wrapper=True, + ) + else: + # Non-transformer layers (e.g. MambaLayer; GatedDeltaNet which does + # accept `sequence_len_offset` is currently always wrapped inside a + # TransformerLayer spec, so it takes the branch above) do not accept + # rotary_pos_emb / sequence_len_offset / padding_mask — pass only + # the common arguments. New layer types that consume any of these + # must add explicit handling here. + output = self.inner_layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + packed_seq_params=packed_seq_params, + ) + + if isinstance(output, tuple): + context = output[1] if len(output) > 1 else None + return output[0], context + return output, None + + def forward( + self, + hidden_states: Tensor, + attention_mask: Tensor, + inference_context: Optional[BaseInferenceContext] = None, + rotary_pos_emb: Optional[Tensor] = None, + sequence_len_offset: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + padding_mask: Optional[Tensor] = None, + mhc_recompute_manager=None, + ) -> Tuple[Tensor, Optional[Tensor]]: + """Run the wrapped hybrid layer through one layer-boundary mHC update.""" + residual = hidden_states + aggregated, h_res, h_post = self.hyper_connection( + hidden_states, mhc_recompute_manager=mhc_recompute_manager + ) + layer_output, context = self._call_inner_layer( + aggregated, + attention_mask, + inference_context, + rotary_pos_emb, + sequence_len_offset, + packed_seq_params, + padding_mask, + ) + # The inner hybrid layer already applied its own local residual/dropout, so + # it returns `aggregated + f(aggregated)`. We feed only the function + # delta `f(aggregated)` into the n-stream BDA so it does not double-count + # the residual that mHC owns. The temporary [s, b, C] tensor here is the + # simplest correct form; a future optimization could fuse the subtraction + # into `fused_h_res_h_post_bda` to avoid the allocation. + # Sanity check: this contract requires the inner layer to preserve shape; + # any mismatch indicates a future layer type is breaking the residual + # assumption and would silently corrupt the n-stream state. + if layer_output.shape != aggregated.shape: + raise RuntimeError( + "HyperConnectionHybridLayer requires inner layers to preserve " + f"hidden-state shape. Got {tuple(layer_output.shape)} from inner layer " + f"vs {tuple(aggregated.shape)} input; layer must add its own residual." + ) + # `fp32_residual_connection=True` may cause some inner layers (e.g., + # MambaLayer) to return `layer_output` in fp32 while `aggregated` is in + # compute dtype; explicitly upcast `aggregated` so the subtraction stays + # in fp32 instead of relying on PyTorch's implicit promotion. + if self.config.fp32_residual_connection and aggregated.dtype != layer_output.dtype: + aggregated = aggregated.to(layer_output.dtype) + layer_delta = layer_output - aggregated + # `dropout_prob=0.0` already disables dropout regardless of training mode; + # `training=self.training` is more semantically accurate than hard-coding + # False during a training-mode forward. + is_last_in_recompute_block = bool( + mhc_recompute_manager is not None + and getattr(mhc_recompute_manager, "is_last_layer_in_recompute_block", False) + ) + mhc_bda_manager = None if is_last_in_recompute_block else mhc_recompute_manager + + hidden_states = self.hyper_connection.fused_h_res_h_post_bda( + h_res, + residual, + h_post, + (layer_delta, None), + dropout_prob=0.0, + training=self.training, + fused=False, + manager=mhc_bda_manager, + ) + # In `HyperConnectionTransformerLayer` the n-stream output stays in compute + # dtype because the post-attention `x` is in compute dtype. In the hybrid + # wrapper, `layer_delta` may be fp32 (when `fp32_residual_connection=True` + # or an inner layer upcasts), so `fused_h_res_h_post_bda`'s `output.to(x.dtype)` + # would leave the result in fp32 and silently propagate fp32 n-stream + # hidden states to every subsequent layer (~2x activation memory). Restore + # the compute-dtype contract here. + if ( + self.config.fp32_residual_connection + and self.config.params_dtype is not None + and hidden_states.dtype != self.config.params_dtype + ): + hidden_states = hidden_states.to(self.config.params_dtype) + return hidden_states, context + + class HybridStack(MegatronModule): """ Constructor for the HybridStack class. @@ -100,6 +268,10 @@ def __init__( self.input_tensor = None self.pg_collection = pg_collection + # Lazily populated mHC recompute layout cache (deterministic from config + # and num_layers); see `_build_mhc_recompute_layer_plan`. + self._mhc_block_end_plan: Optional[List[bool]] = None + assert layer_type_list is not None, ( "layer_type_list must be provided. It should be pre-computed from " "--hybrid-layer-pattern by HybridModel." @@ -172,6 +344,8 @@ def __init__( ) else: raise ValueError("unexpected layer_type") + if self.config.enable_hyper_connections: + layer = HyperConnectionHybridLayer(config=self.config, layer=layer) self.layers.append(layer) # Required for activation recomputation @@ -185,6 +359,18 @@ def __init__( eps=self.config.layernorm_epsilon, ) + if self.config.enable_hyper_connections and self.post_process: + hc_mult = self.config.num_residual_streams + hc_dim = self.config.hidden_size * hc_mult + self.hc_head_fn = nn.Parameter(torch.randn(hc_mult, hc_dim)) + self.hc_head_base = nn.Parameter(torch.zeros(hc_mult)) + self.hc_head_scale = nn.Parameter(torch.ones(1)) + nn.init.xavier_uniform_(self.hc_head_fn) + if self.config.sequence_parallel: + setattr(self.hc_head_fn, 'sequence_parallel', True) + setattr(self.hc_head_base, 'sequence_parallel', True) + setattr(self.hc_head_scale, 'sequence_parallel', True) + def set_input_tensor(self, input_tensor: Tensor): """Set input tensor to be used instead of forward()'s input. @@ -205,6 +391,59 @@ def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int return layer.mamba_state_shapes_per_request() return None + def _compute_mhc_block_end_plan(self) -> List[bool]: + """Compute per-layer block-end markers (deterministic from config).""" + num_layers = len(self.layers) + is_recompute_block_end: List[bool] = [False] * num_layers + if num_layers == 0: + return is_recompute_block_end + mhc_recompute_layer_num = self.config.mhc_recompute_layer_num + for l_no in range(num_layers): + is_last_in_stack = l_no == num_layers - 1 + is_last_in_recompute_block = is_last_in_stack + if mhc_recompute_layer_num is not None: + is_last_in_recompute_block = is_last_in_stack or ( + (l_no + 1) % mhc_recompute_layer_num == 0 + ) + is_recompute_block_end[l_no] = is_last_in_recompute_block + return is_recompute_block_end + + def _build_mhc_recompute_layer_plan( + self, use_mhc_recompute: bool + ) -> Tuple[List[Optional[CheckpointManager]], List[bool]]: + """Pre-build per-layer MHC recompute managers and block-end markers. + + The block-end plan is deterministic from config and cached on the + instance; only the per-block ``CheckpointManager`` instances are + allocated fresh per forward pass (managers are single-use). Mirrors + the caching scheme used by ``TransformerBlock``. + """ + num_layers = len(self.layers) + if not use_mhc_recompute or num_layers == 0: + return [None] * num_layers, [False] * num_layers + + if self._mhc_block_end_plan is None: + self._mhc_block_end_plan = self._compute_mhc_block_end_plan() + is_recompute_block_end = self._mhc_block_end_plan + + layer_managers: List[Optional[CheckpointManager]] = [None] * num_layers + mhc_manager = CheckpointManager() + for l_no in range(num_layers): + layer_managers[l_no] = mhc_manager + if is_recompute_block_end[l_no] and l_no != num_layers - 1: + mhc_manager = CheckpointManager() + return layer_managers, is_recompute_block_end + + @staticmethod + def _finalize_mhc_recompute_layer( + mhc_manager: Optional[CheckpointManager], + hidden_states: Tensor, + is_last_in_recompute_block: bool, + ) -> None: + """Finalize MHC recompute state for the current layer when a block ends.""" + if mhc_manager is not None and is_last_in_recompute_block: + mhc_manager.discard_all_outputs_and_register_unified_recompute(hidden_states) + def forward( self, hidden_states: Union[Tensor, WrappedTensor], @@ -244,6 +483,11 @@ def forward( if isinstance(hidden_states, WrappedTensor): hidden_states = hidden_states.unwrap() + if self.config.enable_hyper_connections and self.pre_process: + hidden_states = HyperConnectionModule.input_expand( + hidden_states, self.config.num_residual_streams + ) + if inference_context and inference_context.is_static_batching(): # NOTE(bnorick): match BaseInferenceContext attributes for # mamba_ssm.utils.generation.BaseInferenceContext, @@ -291,13 +535,29 @@ def get_inner_quant_context(config, layer_number): def get_inner_quant_context(config, layer_number): return nullcontext() + use_mhc_recompute = ( + self.training + and self.config.enable_hyper_connections + and self.config.recompute_granularity == 'selective' + and "mhc" in self.config.recompute_modules + ) + mhc_layer_managers, mhc_is_last_in_recompute_block = self._build_mhc_recompute_layer_plan( + use_mhc_recompute + ) + with outer_fp8_context: - for layer in self.layers: + for l_no, layer in enumerate(self.layers): # Layers have 1-indexed layer numbers attribute. inner_quant_context = get_inner_quant_context(self.config, layer.layer_number - 1) + mhc_manager = mhc_layer_managers[l_no] + if mhc_manager is not None: + mhc_manager.is_last_layer_in_recompute_block = mhc_is_last_in_recompute_block[ + l_no + ] + with inner_quant_context: - if isinstance(layer, TransformerLayer): - hidden_states, _ = layer( + if isinstance(layer, (TransformerLayer, HyperConnectionHybridLayer)): + layer_kwargs = dict( hidden_states=hidden_states, attention_mask=attention_mask, inference_context=inference_context, @@ -306,6 +566,11 @@ def get_inner_quant_context(config, layer_number): packed_seq_params=packed_seq_params, padding_mask=padding_mask, ) + if mhc_manager is not None and isinstance( + layer, HyperConnectionHybridLayer + ): + layer_kwargs["mhc_recompute_manager"] = mhc_manager + hidden_states, _ = layer(**layer_kwargs) else: # MambaLayer, Expert, or MLP hidden_states = layer( hidden_states=hidden_states, @@ -320,6 +585,22 @@ def get_inner_quant_context(config, layer_number): if isinstance(hidden_states, tuple): hidden_states = hidden_states[0] + self._finalize_mhc_recompute_layer( + mhc_manager=mhc_manager, + hidden_states=hidden_states, + is_last_in_recompute_block=mhc_is_last_in_recompute_block[l_no], + ) + + if self.config.enable_hyper_connections and self.post_process: + hidden_states = learned_output_contract( + hidden_states, + self.hc_head_fn, + self.hc_head_base, + self.hc_head_scale, + self.config.num_residual_streams, + self.config.layernorm_epsilon, + ) + # Final layer norm. if self.post_process and self.post_layer_norm: hidden_states = self.final_norm(hidden_states) @@ -354,6 +635,7 @@ def sharded_state_dict( dict: The sharded state dictionary for the current object. """ + sharded_offsets = sharded_offsets or () sharded_state_dict = {} layer_prefix = f'{prefix}layers.' @@ -388,6 +670,20 @@ def sharded_state_dict( ) ) + local_state_dict: dict = {} + self._save_to_state_dict(local_state_dict, '', keep_vars=True) + if local_state_dict: + metadata = ensure_metadata_has_dp_cp_group(metadata) + sharded_state_dict.update( + make_sharded_tensors_for_checkpoint( + local_state_dict, + prefix, + sharded_offsets=sharded_offsets or (), + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], + ) + ) + return sharded_state_dict diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py index c6d3e41aed5..5787049cd8c 100644 --- a/megatron/core/optimizer/__init__.py +++ b/megatron/core/optimizer/__init__.py @@ -939,6 +939,13 @@ def get_megatron_optimizer( model_chunk_offset = 0 ddp_config = model_chunks[0].ddp_config # Use the first model chunk's DDP config if ddp_config.use_megatron_fsdp: + # For no_shard, gradients are replicated across DP ranks after all-reduce, so grad stats + # should only be reduced over TP/PP (model_parallel_group) to avoid inflating the norm. + effective_intra_dist_opt_group = ( + mp_group + if ddp_config.data_parallel_sharding_strategy == 'no_shard' + else intra_dist_opt_group + ) for model_chunk, overlap_param_gather_with_optimizer_step in zip( all_dense_model_chunks, overlap_param_gather_with_optimizer_step_flags ): @@ -960,7 +967,7 @@ def get_megatron_optimizer( data_parallel_group=dp_cp_group, data_parallel_group_gloo=intra_dp_cp_group_gloo, data_parallel_group_idx=model_parallel_rank, - intra_dist_opt_group=intra_dist_opt_group, + intra_dist_opt_group=effective_intra_dist_opt_group, distributed_optimizer_instance_id=distributed_optimizer_instance_id, pg_collection=pg_collection, ) diff --git a/megatron/core/pipeline_parallel/combined_1f1b.py b/megatron/core/pipeline_parallel/combined_1f1b.py index b1ebbb876ff..81524363993 100644 --- a/megatron/core/pipeline_parallel/combined_1f1b.py +++ b/megatron/core/pipeline_parallel/combined_1f1b.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import contextlib from contextlib import nullcontext @@ -21,6 +21,17 @@ Shape = Union[List[int], torch.Size] +def _release_tensor_storage(tensors): + """Release tensor storage after all backward users are done.""" + if tensors is None: + return + + for tensor in tensors: + if isinstance(tensor, torch.Tensor) and tensor.is_cuda: + tensor.record_stream(torch.cuda.current_stream()) + tensor.untyped_storage().resize_(0) + + def combined_1f1b_schedule_for_no_pipelining( forward_step_func, data_iterator, @@ -405,6 +416,7 @@ def forward_backward_step(): # backward preprocess, the same as the backward_step() unwrap_input_tensor_grad = False b_schedule_plan = None + loss_node_inputs_to_release = None if b_model is not None: # Retain the grad on the input_tensor. if not isinstance(b_input_tensor, list): @@ -432,6 +444,8 @@ def forward_backward_step(): # Backward pass for loss function torch.autograd.backward(b_output_tensor[0], grad_tensors=b_output_tensor_grad[0]) b_output_tensor_grad[0] = loss_node.get_grad() + loss_node_inputs_to_release = loss_node.inputs + loss_node._release_state() # If fp8_recipe is delayed, wrap the entire pass with get_fp8_context(), # otherwise do nothing extra at the outer level @@ -454,6 +468,7 @@ def forward_backward_step(): post_forward=post_forward, post_backward=post_backward, ) + _release_tensor_storage(loss_node_inputs_to_release) # forward post process num_tokens = None diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index c6c6ed071ad..2b27f367b00 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -321,6 +321,23 @@ def forward_step_calc_loss( else: MTPLossAutoScaler.set_loss_scale(loss_scale / num_microbatches) + # Set the loss scale for the DSA indexer loss. + if hasattr(config, 'dsa_indexer_loss_coeff') and config.dsa_indexer_loss_coeff is not None: + from megatron.core.transformer.experimental_attention_variant.dsa import ( + DSAIndexerLossAutoScaler, + ) + + device = get_tensor_device(output_tensor) + loss_scale = ( + config.grad_scale_func(torch.ones(1, device=device)) + if config.grad_scale_func is not None + else torch.ones(1, device=device) + ) + if config.calculate_per_token_loss: + DSAIndexerLossAutoScaler.set_loss_scale(loss_scale) + else: + DSAIndexerLossAutoScaler.set_loss_scale(loss_scale / num_microbatches) + return output_tensor, num_tokens diff --git a/megatron/core/ssm/gated_delta_net.py b/megatron/core/ssm/gated_delta_net.py index f9b923632f5..e6f728b99da 100644 --- a/megatron/core/ssm/gated_delta_net.py +++ b/megatron/core/ssm/gated_delta_net.py @@ -17,6 +17,12 @@ from megatron.core.dist_checkpointing import ShardedTensor from megatron.core.dist_checkpointing.mapping import ReplicaId, ShardedTensorFactory from megatron.core.fp8_utils import get_fp8_align_size +from megatron.core.fusions.fused_mega_pre_gated_delta_rule import ( + fused_mega_pre_gated_delta_rule, +) +from megatron.core.fusions.fused_pre_gated_delta_rule import ( + fused_streamed_pre_gated_delta_rule, +) from megatron.core.inference.contexts import BaseInferenceContext from megatron.core.jit import jit_fuser from megatron.core.packed_seq_params import PackedSeqParams @@ -122,6 +128,11 @@ def __init__( self.cp_size = self.pg_collection.cp.size() self.tp_size = self.pg_collection.tp.size() self.sp_size = self.tp_size if config.sequence_parallel else 1 + self.pre_gated_delta_rule_impl = config.pre_gated_delta_rule_impl + if self.pre_gated_delta_rule_impl != "unfused": + assert ( + self.cp_size == 1 + ), "Fused pre_gated_delta_rule does not support context parallelism yet." # Attributes from config self.config = config @@ -300,6 +311,11 @@ def forward( # TODO: support inference raise NotImplementedError("GDN does not support inference for now.") + if self.pre_gated_delta_rule_impl != "unfused": + assert ( + self.cp_size == 1 + ), "Fused pre_gated_delta_rule does not support context parallelism yet." + if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': assert batch == 1, "Packed sequence expects batch dimension to be 1" assert ( @@ -374,6 +390,84 @@ def forward( ], ) + if self.pre_gated_delta_rule_impl == "fused_streamed": + nvtx_range_push(suffix="fused_streamed_pre_gated_delta_rule") + seq_idx = ( + packed_seq_params.seq_idx + if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd' + else None + ) + query, key, value, gate, beta, g = self._fused_streamed_pre_gated_delta_rule( + qkvzba, cu_seqlens_q=cu_seqlens_q, seq_idx=seq_idx + ) + nvtx_range_pop(suffix="fused_streamed_pre_gated_delta_rule") + elif self.pre_gated_delta_rule_impl == "fused_mega": + nvtx_range_push(suffix="fused_mega_pre_gated_delta_rule") + seq_idx = ( + packed_seq_params.seq_idx + if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd' + else None + ) + query, key, value, gate, beta, g = self._fused_mega_pre_gated_delta_rule( + qkvzba, cu_seqlens_q=cu_seqlens_q, seq_idx=seq_idx + ) + nvtx_range_pop(suffix="fused_mega_pre_gated_delta_rule") + else: + nvtx_range_push(suffix="pre_gated_delta_rule") + query, key, value, gate, beta, g = self.pre_gated_delta_rule( + qkvzba, batch, seq_len, cu_seqlens_q=cu_seqlens_q + ) + nvtx_range_pop(suffix="pre_gated_delta_rule") + + nvtx_range_push(suffix="gated_delta_rule") + core_attn_out, last_recurrent_state = self.gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=None, + output_final_state=False, + use_qk_l2norm_in_kernel=False, + cu_seqlens=cu_seqlens_q, + ) + nvtx_range_pop(suffix="gated_delta_rule") + + # RMSNorm + nvtx_range_push(suffix="gated_norm") + norm_out = self._apply_gated_norm(core_attn_out, gate) + nvtx_range_pop(suffix="gated_norm") + + # Transpose: b s x --> s b x + # From bshd back to sbhd format + norm_out = norm_out.reshape(batch, seq_len, -1) + norm_out = norm_out.transpose(0, 1).contiguous() + + # CP all to all: HP to CP + if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': + unpacked_norm_out = _unpack_sequence(norm_out, cu_seqlens_q, dim=0) + outputs = [] + for norm_out_i in unpacked_norm_out: + norm_out_i = tensor_a2a_hp2cp( + norm_out_i, seq_dim=0, head_dim=-1, cp_group=self.pg_collection.cp + ) + outputs.append(norm_out_i) + norm_out = torch.cat(outputs, dim=0) + else: + norm_out = tensor_a2a_hp2cp( + norm_out, seq_dim=0, head_dim=-1, cp_group=self.pg_collection.cp + ) + + # Output projection + nvtx_range_push(suffix="out_proj") + out, out_bias = self.out_proj(norm_out) + nvtx_range_pop(suffix="out_proj") + + return out, out_bias + + def pre_gated_delta_rule(self, qkvzba, batch, seq_len, cu_seqlens_q=None): + """Prepare QKV, gate, beta, and decay tensors before the gated delta rule.""" + # Transpose: s b x --> b s x # From sbhd to bshd format qkvzba = qkvzba.transpose(0, 1) @@ -459,51 +553,45 @@ def forward( g, beta = self._compute_g_and_beta(A_log_local_cp, dt_bias_local_cp, alpha, beta) nvtx_range_pop(suffix="g_and_beta") - nvtx_range_push(suffix="gated_delta_rule") - core_attn_out, last_recurrent_state = self.gated_delta_rule( - query, - key, - value, - g=g, - beta=beta, - initial_state=None, - output_final_state=False, - use_qk_l2norm_in_kernel=False, - cu_seqlens=cu_seqlens_q, - ) - nvtx_range_pop(suffix="gated_delta_rule") + return query, key, value, gate, beta, g - # RMSNorm - nvtx_range_push(suffix="gated_norm") - norm_out = self._apply_gated_norm(core_attn_out, gate) - nvtx_range_pop(suffix="gated_norm") - - # Transpose: b s x --> s b x - # From bshd back to sbhd format - norm_out = norm_out.reshape(batch, seq_len, -1) - norm_out = norm_out.transpose(0, 1).contiguous() + def _fused_streamed_pre_gated_delta_rule(self, qkvzba, cu_seqlens_q=None, seq_idx=None): + """Call the streamed fused pre-GDR wrapper.""" - # CP all to all: HP to CP - if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': - unpacked_norm_out = _unpack_sequence(norm_out, cu_seqlens_q, dim=0) - outputs = [] - for norm_out_i in unpacked_norm_out: - norm_out_i = tensor_a2a_hp2cp( - norm_out_i, seq_dim=0, head_dim=-1, cp_group=self.pg_collection.cp - ) - outputs.append(norm_out_i) - norm_out = torch.cat(outputs, dim=0) - else: - norm_out = tensor_a2a_hp2cp( - norm_out, seq_dim=0, head_dim=-1, cp_group=self.pg_collection.cp - ) + assert self.cp_size == 1, "Fused pre_gated_delta_rule does not support CP yet." + return fused_streamed_pre_gated_delta_rule( + qkvzba, + self.conv1d.weight, + self.conv1d.bias if self.conv_bias else None, + self.A_log, + self.dt_bias, + num_key_heads=self.qk_dim_local_tp // self.key_head_dim, + num_value_heads=self.v_dim_local_tp // self.value_head_dim, + key_head_dim=self.key_head_dim, + value_head_dim=self.value_head_dim, + use_qk_l2norm=self.use_qk_l2norm, + cu_seqlens=cu_seqlens_q, + seq_idx=seq_idx, + ) - # Output projection - nvtx_range_push(suffix="out_proj") - out, out_bias = self.out_proj(norm_out) - nvtx_range_pop(suffix="out_proj") + def _fused_mega_pre_gated_delta_rule(self, qkvzba, cu_seqlens_q=None, seq_idx=None): + """Call the mega fused pre-GDR wrapper.""" - return out, out_bias + assert self.cp_size == 1, "Fused pre_gated_delta_rule does not support CP yet." + return fused_mega_pre_gated_delta_rule( + qkvzba, + self.conv1d.weight, + self.conv1d.bias if self.conv_bias else None, + self.A_log, + self.dt_bias, + num_key_heads=self.qk_dim_local_tp // self.key_head_dim, + num_value_heads=self.v_dim_local_tp // self.value_head_dim, + key_head_dim=self.key_head_dim, + value_head_dim=self.value_head_dim, + use_qk_l2norm=self.use_qk_l2norm, + cu_seqlens=cu_seqlens_q, + seq_idx=seq_idx, + ) @jit_fuser def _apply_gated_norm(self, x, gate): diff --git a/megatron/core/transformer/experimental_attention_variant/csa.py b/megatron/core/transformer/experimental_attention_variant/csa.py index 547c1828a95..2e450de4826 100644 --- a/megatron/core/transformer/experimental_attention_variant/csa.py +++ b/megatron/core/transformer/experimental_attention_variant/csa.py @@ -20,6 +20,12 @@ fused_qk_topk_naive, rotate_activation, ) +from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + build_flat_topk_idxs, + dsa_sparse_attn, + fused_indexer_sparse_attn, + indexer_topk, +) from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.transformer_config import TransformerConfig @@ -103,27 +109,37 @@ def _apply_rope( total_seq_len = rotary_seq_len else: total_seq_len = rotary_seq_len * ratio + # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's + # concentration factor (mscale) is NOT part of the DSv4 model contract -- + # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0 + # regardless of which rotary class is in use. mscale = 1.0 rotary_pos_cos = None rotary_pos_sin = None - if config.rope_type == "rope": - rotary_pos_emb = rotary_pos_emb_module(total_seq_len, packed_seq=False) - mscale = 1.0 + if config.apply_rope_fusion: + # ``mscale=1.0`` keeps the cached cos/sin free of yarn's + # concentration factor so the fused kernel sees the same + # rotation as the unfused split-rotate path (DSv4 "pure + # rotation" contract). + rotary_pos_cos, rotary_pos_sin = rotary_pos_emb_module.get_cached_cos_sin( + total_seq_len, dtype=x.dtype, packed_seq=False, mscale=mscale + ) + rotary_pos_emb = None + assert ( + fused_mla_rope_inplace is not None + ), "Fused MLA RoPE apply is not imported successfully" else: - if config.apply_rope_fusion: - rotary_pos_cos, rotary_pos_sin = rotary_pos_emb_module.get_cached_cos_sin( - total_seq_len, dtype=x.dtype, packed_seq=False - ) - rotary_pos_emb = None - assert ( - fused_mla_rope_inplace is not None - ), "Fused MLA RoPE apply is not imported successfully" + # ``DSv4HybridAttention`` instantiates ``YarnRotaryEmbedding`` + # whenever ``compress_ratio > 1`` (regardless of ``config.rope_type``); + # its ``forward`` returns ``(emb, mscale)``. Base ``RotaryEmbedding`` + # returns a single tensor. Unpack either form uniformly; the + # caller-side ``mscale=1.0`` keeps the yarn concentration factor + # out of the rotation. + result = rotary_pos_emb_module(total_seq_len, packed_seq=False) + if isinstance(result, tuple): + rotary_pos_emb = result[0] else: - rotary_pos_emb, mscale = rotary_pos_emb_module(total_seq_len, packed_seq=False) - # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's - # concentration factor (mscale) is NOT part of the DSv4 model contract -- - # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0. - mscale = 1.0 + rotary_pos_emb = result if rotary_pos_emb is not None and ratio > 1: rotary_pos_emb = rotary_pos_emb[:total_seq_len:ratio][:rotary_seq_len] if rotary_pos_cos is not None and ratio > 1: @@ -596,7 +612,7 @@ def __init__( softmax_scale = config.v_head_dim**-0.5 self.softmax_scale = softmax_scale - self.force_unfused_dsa = getattr(config, 'force_unfused_dsa', True) + self.apply_dsa_kernel_fusion = config.apply_dsa_kernel_fusion # Learnable attention sink per head self.attn_sink = nn.Parameter(torch.zeros(self.n_local_heads, dtype=torch.float32)) @@ -631,6 +647,265 @@ def __init__( else: self.indexer = None + # ------------------------------------------------------------------ + # Private helpers – each owns one logical slice of the forward pass. + # ------------------------------------------------------------------ + + def _build_kv_full( + self, kv: torch.Tensor, x: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], int]: + """Concatenate original KV with compressed KV (if applicable). + + Returns: + kv_full: [n_kv, b, v_head_dim] original + compressed KV. + compressed_kv: [n_compressed, b, v_head_dim] or None. + n_compressed: number of compressed positions (0 when unused). + """ + if self.compressor is not None and self.compress_ratio > 1: + compressed_kv = self.compressor(x) + if compressed_kv is not None: + kv_full = torch.cat([kv, compressed_kv], dim=0) + n_compressed = compressed_kv.size(0) + else: + kv_full = kv + compressed_kv = None + n_compressed = 0 + else: + kv_full = kv + compressed_kv = None + n_compressed = 0 + return kv_full, compressed_kv, n_compressed + + def _forward_unfused_csa( + self, + query: torch.Tensor, + x: torch.Tensor, + qr: torch.Tensor, + kv_full: torch.Tensor, + compressed_kv: Optional[torch.Tensor], + n_compressed: int, + offset: int, + window_idxs: torch.Tensor, + packed_seq_params: Optional[PackedSeqParams], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """PyTorch fallback path (no fused kernels). + + Returns ``(output, indexer_loss)``. + """ + sq, b, np, hn = query.size() + indexer_loss = None + + if self.compress_ratio > 1 and n_compressed > 0: + nvtx_range_push("compressed_indices") + if self.indexer is not None: + x_det = x.detach() + qr_det = qr.detach() + + causal_mask = ( + torch.arange(n_compressed, device=x.device).unsqueeze(0).expand(sq, -1) + ) + positions = torch.arange(1, sq + 1, device=x.device).unsqueeze(1) + causal_mask = ( + torch.where(causal_mask >= positions // self.compress_ratio, float("-inf"), 0.0) + .unsqueeze(0) + .expand(b, -1, -1) + ) # [b, sq, n_compressed] + + if self.training and torch.is_grad_enabled(): + q_indexer, k_indexer, weights_indexer = self.indexer.forward_before_topk( + x_det, qr_det, packed_seq_params + ) + indexer_loss_coeff = getattr(self.config, 'dsa_indexer_loss_coeff', 0.0) + key_for_loss = compressed_kv.unsqueeze(2).expand(-1, -1, np, -1) + # ``FusedDSAIndexerLoss`` does not accept a separate + # indexer_softmax_scale; apply it here via the + # weights-scaling trick so the effective weights match + # the pre-scale-split behaviour. + weights_for_unfused = weights_indexer.float() * self.indexer.softmax_scale + topk_indices_compressed, indexer_loss = FusedDSAIndexerLoss.apply( + q_indexer, + weights_for_unfused, + k_indexer, + query.detach(), + key_for_loss.detach(), + self.softmax_scale, + min(self.indexer.index_topk, n_compressed), + indexer_loss_coeff, + causal_mask, + getattr(self.config, "dsa_indexer_use_sparse_loss", True), + self.indexer.pg_collection, + self.config.calculate_per_token_loss, + ) + if indexer_loss_coeff > 0: + DSAIndexerLossLoggingHelper.save_loss_to_tracker( + loss=indexer_loss, + layer_number=self.layer_number, + num_layers=self.config.num_layers + (self.config.mtp_num_layers or 0), + ) + else: + _, topk_indices_compressed = self.indexer( + x_det, qr_det, mask=causal_mask, packed_seq_params=packed_seq_params + ) + + n_valid_per_pos = positions // self.compress_ratio # [sq, 1] + valid = topk_indices_compressed < n_valid_per_pos + compress_topk_idxs = torch.where( + valid, topk_indices_compressed + offset, torch.tensor(-1, device=x.device) + ) + else: + compress_topk_idxs = get_compress_topk_idxs( + self.compress_ratio, b, sq, offset, query.device + ) + + topk_idxs = torch.cat([window_idxs, compress_topk_idxs], dim=-1) + nvtx_range_pop("compressed_indices") + else: + topk_idxs = window_idxs + + topk_idxs = topk_idxs.int() + + nvtx_range_push("sparse_attn_kernel") + output = unfused_compressed_sparse_attn( + query, kv_full, self.attn_sink.float(), topk_idxs, self.softmax_scale + ) + nvtx_range_pop("sparse_attn_kernel") + return output, indexer_loss + + def _forward_fused_no_indexer( + self, + query: torch.Tensor, + kv_full: torch.Tensor, + n_compressed: int, + offset: int, + window_idxs: torch.Tensor, + ) -> torch.Tensor: + """Path A: fused sparse attn with window or deterministic compressed indices.""" + sq, b, np, hn = query.size() + + nvtx_range_push("compressed_indices") + if self.compress_ratio > 1 and n_compressed > 0: + compress_topk_idxs = get_compress_topk_idxs( + self.compress_ratio, b, sq, offset, query.device + ) + flat_idxs, _ = build_flat_topk_idxs( + window_idxs, compress_topk_idxs, batch_size=b, seqlen_kv=kv_full.shape[0] + ) + else: + flat_idxs, _ = build_flat_topk_idxs( + window_idxs, batch_size=b, seqlen_kv=kv_full.shape[0] + ) + nvtx_range_pop("compressed_indices") + + nvtx_range_push("sparse_attn_kernel") + output = dsa_sparse_attn( + query, kv_full, self.attn_sink.float(), flat_idxs, self.softmax_scale + ) + nvtx_range_pop("sparse_attn_kernel") + return output + + def _forward_fused_indexer_inference( + self, + query: torch.Tensor, + x: torch.Tensor, + qr: torch.Tensor, + kv_full: torch.Tensor, + n_compressed: int, + offset: int, + window_idxs: torch.Tensor, + packed_seq_params: Optional[PackedSeqParams], + ) -> torch.Tensor: + """Path C: separate indexer forward (no loss) + fused sparse attn (compact).""" + b = query.size(1) + + nvtx_range_push("compressed_indices") + x_det = x.detach() + qr_det = qr.detach() + q_indexer, k_indexer, weights_indexer = self.indexer.forward_before_topk( + x_det, qr_det, packed_seq_params + ) + topk_indices_cmp, _ = indexer_topk( + q_indexer, + k_indexer, + weights_indexer, + min(self.indexer.index_topk, n_compressed), + self.compress_ratio, + indexer_softmax_scale=self.indexer.softmax_scale, + ) + compress_topk_idxs = torch.where(topk_indices_cmp >= 0, topk_indices_cmp + offset, -1) + flat_idxs, flat_tlen = build_flat_topk_idxs( + window_idxs, compress_topk_idxs, batch_size=b, seqlen_kv=kv_full.shape[0], compact=True + ) + nvtx_range_pop("compressed_indices") + + nvtx_range_push("sparse_attn_kernel") + output = dsa_sparse_attn( + query, + kv_full, + self.attn_sink.float(), + flat_idxs, + self.softmax_scale, + topk_length=flat_tlen, + ) + nvtx_range_pop("sparse_attn_kernel") + return output + + def _forward_fused_indexer_training( + self, + query: torch.Tensor, + x: torch.Tensor, + qr: torch.Tensor, + kv_full: torch.Tensor, + n_compressed: int, + offset: int, + window_idxs: torch.Tensor, + packed_seq_params: Optional[PackedSeqParams], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Path B: fused indexer (with loss) + fused sparse attn. + + Returns ``(output, indexer_loss)``. + """ + nvtx_range_push("compressed_indices") + x_det = x.detach() + qr_det = qr.detach() + q_indexer, k_indexer, weights_indexer = self.indexer.forward_before_topk( + x_det, qr_det, packed_seq_params + ) + nvtx_range_pop("compressed_indices") + + indexer_loss_coeff = self.config.dsa_indexer_loss_coeff or 0.0 + + nvtx_range_push("sparse_attn_kernel") + output, indexer_loss = fused_indexer_sparse_attn( + query, + kv_full, + self.attn_sink.float(), + window_idxs, + q_indexer, + k_indexer, + weights_indexer, + min(self.indexer.index_topk, n_compressed), + self.compress_ratio, + self.softmax_scale, + self.indexer.softmax_scale, + indexer_loss_coeff, + sparse_loss=getattr(self.config, "dsa_indexer_use_sparse_loss", True), + kv_offset=offset, + calculate_per_token_loss=self.config.calculate_per_token_loss, + ) + nvtx_range_pop("sparse_attn_kernel") + + if indexer_loss_coeff > 0: + DSAIndexerLossLoggingHelper.save_loss_to_tracker( + loss=indexer_loss, + layer_number=self.layer_number, + num_layers=self.config.num_layers + (self.config.mtp_num_layers or 0), + ) + return output, indexer_loss + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + def forward( self, query: torch.Tensor, @@ -663,115 +938,43 @@ def forward( sq, b, np, hn = query.size() - # --- Step 1: Prepare single-head KV (squeeze singleton head dim) --- kv = key.squeeze(-2) # [sq, b, 1, v_head_dim] -> [sq, b, v_head_dim] - - # --- Step 2: Compression --- - if self.compressor is not None and self.compress_ratio > 1: - compressed_kv = self.compressor(x) # [n_compressed, b, v_head_dim] - if compressed_kv is not None: - kv_full = torch.cat([kv, compressed_kv], dim=0) - n_compressed = compressed_kv.size(0) - else: - kv_full = kv - n_compressed = 0 - else: - kv_full = kv - n_compressed = 0 - + kv_full, compressed_kv, n_compressed = self._build_kv_full(kv, x) offset = sq # compressed indices start after original positions - - # --- Step 3: Window indices --- window_idxs = get_window_topk_idxs(self.window_size, b, sq, query.device) - # --- Step 4: Compressed indices --- - indexer_loss = None - - if self.force_unfused_dsa: - if self.compress_ratio > 1 and n_compressed > 0: - nvtx_range_push("compressed_indices") - if self.indexer is not None: - x_det = x.detach() - qr_det = qr.detach() - - causal_mask = ( - torch.arange(n_compressed, device=x.device).unsqueeze(0).expand(sq, -1) - ) - positions = torch.arange(1, sq + 1, device=x.device).unsqueeze(1) - causal_mask = ( - torch.where( - causal_mask >= positions // self.compress_ratio, float("-inf"), 0.0 - ) - .unsqueeze(0) - .expand(b, -1, -1) - ) # [b, sq, n_compressed] - - if self.training and torch.is_grad_enabled(): - q_indexer, k_indexer, weights_indexer = self.indexer.forward_before_topk( - x_det, qr_det, packed_seq_params - ) - indexer_loss_coeff = getattr(self.config, 'dsa_indexer_loss_coeff', 0.0) - # compressed_kv is [n, b, hn]; expand to [n, b, np, hn] for loss - key_for_loss = compressed_kv.unsqueeze(2).expand(-1, -1, np, -1) - # ``FusedDSAIndexerLoss`` does not accept a separate - # indexer_softmax_scale; apply it here via the - # weights-scaling trick so the effective weights match - # the pre-scale-split behaviour. - weights_for_unfused = weights_indexer * self.indexer.softmax_scale - topk_indices_compressed, indexer_loss = FusedDSAIndexerLoss.apply( - q_indexer, - weights_for_unfused, - k_indexer, - query.detach(), - key_for_loss.detach(), - self.softmax_scale, - min(self.indexer.index_topk, n_compressed), - indexer_loss_coeff, - causal_mask, - getattr(self.config, "dsa_indexer_use_sparse_loss", True), - self.indexer.pg_collection, - ) - if indexer_loss_coeff > 0: - DSAIndexerLossLoggingHelper.save_loss_to_tracker( - loss=indexer_loss, - layer_number=self.layer_number, - num_layers=self.config.num_layers - + (self.config.mtp_num_layers or 0), - ) - else: - _, topk_indices_compressed = self.indexer( - x_det, qr_det, mask=causal_mask, packed_seq_params=packed_seq_params - ) - - n_valid_per_pos = positions // self.compress_ratio # [sq, 1] - valid = topk_indices_compressed < n_valid_per_pos - compress_topk_idxs = torch.where( - valid, topk_indices_compressed + offset, torch.tensor(-1, device=x.device) - ) - else: - compress_topk_idxs = get_compress_topk_idxs( - self.compress_ratio, b, sq, offset, query.device - ) - - topk_idxs = torch.cat([window_idxs, compress_topk_idxs], dim=-1) - nvtx_range_pop("compressed_indices") - else: - topk_idxs = window_idxs + has_indexer_compressed = ( + self.compress_ratio > 1 and n_compressed > 0 and self.indexer is not None + ) - topk_idxs = topk_idxs.int() + indexer_loss = None - # --- Step 5: Sparse attention --- - nvtx_range_push("sparse_attn_kernel") - output = unfused_compressed_sparse_attn( - query, kv_full, self.attn_sink.float(), topk_idxs, self.softmax_scale + if not self.apply_dsa_kernel_fusion: + output, indexer_loss = self._forward_unfused_csa( + query, + x, + qr, + kv_full, + compressed_kv, + n_compressed, + offset, + window_idxs, + packed_seq_params, + ) + elif has_indexer_compressed and self.training and torch.is_grad_enabled(): + output, indexer_loss = self._forward_fused_indexer_training( + query, x, qr, kv_full, n_compressed, offset, window_idxs, packed_seq_params + ) + elif has_indexer_compressed: + output = self._forward_fused_indexer_inference( + query, x, qr, kv_full, n_compressed, offset, window_idxs, packed_seq_params ) - nvtx_range_pop("sparse_attn_kernel") - else: - raise ValueError("Fused path is not supported for CompressedSparseAttention") + output = self._forward_fused_no_indexer( + query, kv_full, n_compressed, offset, window_idxs + ) - # --- Step 6: Attach indexer loss --- - if indexer_loss is not None and self.training and torch.is_grad_enabled(): + if indexer_loss is not None: output = DSAIndexerLossAutoScaler.apply(output, indexer_loss) nvtx_range_pop("compressed_sparse_attn") diff --git a/megatron/core/transformer/experimental_attention_variant/deepseek_v4_hybrid_attention.py b/megatron/core/transformer/experimental_attention_variant/deepseek_v4_hybrid_attention.py index 0e0a69cb6e9..0c15ac5ac94 100644 --- a/megatron/core/transformer/experimental_attention_variant/deepseek_v4_hybrid_attention.py +++ b/megatron/core/transformer/experimental_attention_variant/deepseek_v4_hybrid_attention.py @@ -114,17 +114,21 @@ def __init__( compress_ratio = self.config.csa_compress_ratios[layer_idx] else: compress_ratio = self.config.csa_compress_ratios[layer_number - 1] - rope_base = self.config.rotary_base - if compress_ratio > 1: - rope_base = self.config.csa_compress_rotary_base - if self.config.rope_type == "rope": + use_compressed_yarn = compress_ratio > 1 + rope_base = ( + self.config.csa_compress_rotary_base if use_compressed_yarn else self.config.rotary_base + ) + self._dsv4_compress_ratio = compress_ratio + self._dsv4_rope_base = rope_base + self._dsv4_uses_yarn_rope = use_compressed_yarn + if not use_compressed_yarn: self.rotary_pos_emb = RotaryEmbedding( self.config.qk_pos_emb_head_dim, rotary_percent=self.config.rotary_percent, rotary_base=rope_base, cp_group=self.pg_collection.cp, ) - elif self.config.rope_type == "yarn": + else: self.rotary_pos_emb = YarnRotaryEmbedding( self.config.qk_pos_emb_head_dim, rotary_base=rope_base, @@ -136,11 +140,6 @@ def __init__( mscale_all_dim=self.config.mscale_all_dim, cp_group=self.pg_collection.cp, ) - else: - raise ValueError( - f"Unsupported RoPE type: {self.config.rope_type}, supported types are " - "'rope' and 'yarn'" - ) core_attn_extra_kwargs = { "rotary_pos_emb": self.rotary_pos_emb, @@ -314,27 +313,28 @@ def forward( else: cu_seqlens_kv = None rope_seqlen = seq_len + # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's + # concentration factor (mscale) is NOT part of the DSv4 model contract -- + # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0. mscale = 1.0 rotary_pos_cos = None rotary_pos_sin = None - if self.config.rope_type == "rope": - rotary_pos_emb = self.rotary_pos_emb(rope_seqlen, packed_seq=packed_seq) + if self.config.apply_rope_fusion: + # ``mscale=1.0`` strips yarn's concentration factor from the + # cached cos/sin so the fused kernel matches the unfused + # path's forced ``mscale=1.0`` (DSv4 "pure rotation"). + rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb.get_cached_cos_sin( + rope_seqlen, dtype=hidden_states.dtype, packed_seq=packed_seq, mscale=mscale + ) + rotary_pos_emb = None + assert inference_context is None, "Inference with MLA RoPE fusion is not supported" + assert ( + fused_mla_rope_inplace is not None + ), "Fused MLA RoPE apply is not imported successfully" + elif self._dsv4_uses_yarn_rope: + rotary_pos_emb, _ = self.rotary_pos_emb(rope_seqlen, packed_seq=packed_seq) else: - if self.config.apply_rope_fusion: - rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb.get_cached_cos_sin( - rope_seqlen, dtype=hidden_states.dtype, packed_seq=packed_seq - ) - rotary_pos_emb = None - assert inference_context is None, "Inference with MLA RoPE fusion is not supported" - assert ( - fused_mla_rope_inplace is not None - ), "Fused MLA RoPE apply is not imported successfully" - else: - rotary_pos_emb, mscale = self.rotary_pos_emb(rope_seqlen, packed_seq=packed_seq) - # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's - # concentration factor (mscale) is NOT part of the DSv4 model contract -- - # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0. - mscale = 1.0 + rotary_pos_emb = self.rotary_pos_emb(rope_seqlen, packed_seq=packed_seq) if self.config.apply_rope_fusion: core_attn_out = fused_mla_rope_inplace( core_attn_out, @@ -514,28 +514,29 @@ def get_query_key_value_tensors( ) # rotary_pos_emb:[s, b, 1, 64] + # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's + # concentration factor (mscale) is NOT part of the DSv4 model contract -- + # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0. mscale = 1.0 rotary_pos_cos = None rotary_pos_sin = None packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == 'thd' - if self.config.rope_type == "rope": - rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq) + if self.config.apply_rope_fusion: + # ``mscale=1.0`` strips yarn's concentration factor from the + # cached cos/sin so the fused kernel matches the unfused + # path's forced ``mscale=1.0`` (DSv4 "pure rotation"). + rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb.get_cached_cos_sin( + rotary_seq_len, dtype=hidden_states.dtype, packed_seq=packed_seq, mscale=mscale + ) + rotary_pos_emb = None + assert inference_context is None, "Inference with MLA RoPE fusion is not supported" + assert ( + fused_mla_rope_inplace is not None + ), "Fused MLA RoPE apply is not imported successfully" + elif self._dsv4_uses_yarn_rope: + rotary_pos_emb, _ = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq) else: - if self.config.apply_rope_fusion: - rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb.get_cached_cos_sin( - rotary_seq_len, dtype=hidden_states.dtype, packed_seq=packed_seq - ) - rotary_pos_emb = None - assert inference_context is None, "Inference with MLA RoPE fusion is not supported" - assert ( - fused_mla_rope_inplace is not None - ), "Fused MLA RoPE apply is not imported successfully" - else: - rotary_pos_emb, mscale = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq) - # DSv4 reference (DS-Inf) RoPE is pure rotation (norm-preserving). Yarn's - # concentration factor (mscale) is NOT part of the DSv4 model contract -- - # the model relies on Q/KV RMS-norm + unit-magnitude rotation. Force 1.0. - mscale = 1.0 + rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq) if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': if packed_seq_params.cu_seqlens_q_padded is not None: diff --git a/megatron/core/transformer/experimental_attention_variant/dsa.py b/megatron/core/transformer/experimental_attention_variant/dsa.py index 94f0fae781c..5ee9d07b886 100644 --- a/megatron/core/transformer/experimental_attention_variant/dsa.py +++ b/megatron/core/transformer/experimental_attention_variant/dsa.py @@ -196,6 +196,7 @@ def compute_dsa_indexer_loss( sparse_loss: bool, pg_collection: ProcessGroupCollection, causal_mask_override: Optional[torch.Tensor] = None, + calculate_per_token_loss: bool = False, ) -> torch.Tensor: """ Compute KL divergence loss between index_scores and true attention_scores. @@ -216,6 +217,10 @@ def compute_dsa_indexer_loss( sparse_loss: bool, whether to use sparse indexer loss. If True, only the topk indices will be used to compute the loss. pg_collection: Process group collection, must have TP process group. + causal_mask_override: Optional mask used by compressed KV paths. + calculate_per_token_loss: If True, return a raw local sum so the global + token divisor can be applied by finalize_model_grads. If False, keep + the historical local BSHD average over ``batch * seqlen`` rows. Returns: index_loss: KL divergence loss (scalar). @@ -308,7 +313,11 @@ def compute_dsa_indexer_loss( # [b, sq, sk] -> [b, sq] -> [1] # Each token has same weight in the loss. - kl_div = kl_per_element.sum(dim=-1).mean() + kl_per_row = kl_per_element.sum(dim=-1) + if calculate_per_token_loss: + kl_div = kl_per_row.sum() + else: + kl_div = kl_per_row.mean() # Scale by coefficient. indexer_loss = kl_div * loss_coeff @@ -388,7 +397,18 @@ def fused_qk_topk_naive( def fwd_fused_indexer_loss_naive( - q, weights, k, query, key, topk, softmax_scale, loss_coeff, mask, sparse_loss, pg_collection + q, + weights, + k, + query, + key, + topk, + softmax_scale, + loss_coeff, + mask, + sparse_loss, + pg_collection, + calculate_per_token_loss, ): """Naive implementation of forward pass for indexer loss.""" index_scores, topk_indices = fused_qk_topk_naive(q, k, weights, topk, mask) @@ -403,6 +423,7 @@ def fwd_fused_indexer_loss_naive( sparse_loss, pg_collection, causal_mask_override=mask, + calculate_per_token_loss=calculate_per_token_loss, ) return topk_indices, indexer_loss @@ -421,6 +442,7 @@ def bwd_fused_indexer_loss_naive( grad_loss, pg_collection, causal_mask_override=None, + calculate_per_token_loss=False, ): """Naive implementation of backward pass for indexer loss.""" index_scores = _compute_index_scores(q, weights, k) # [B, Sq, Sk] @@ -520,11 +542,15 @@ def bwd_fused_indexer_loss_naive( del attention_scores_sum # Backward through loss = kl_div * loss_coeff - # where kl_div = kl_per_element.sum(dim=-1).mean() + # where kl_div is either kl_per_element.sum(dim=-1).mean() or the raw + # local sum when calculate_per_token_loss=True. grad_kl_div = grad_loss * loss_coeff # scalar - # Backward through mean: distribute gradient equally - grad_kl_per_row = grad_kl_div / (b * sq) # scalar value for each row + if calculate_per_token_loss: + grad_kl_per_row = grad_kl_div + else: + # Backward through mean: distribute gradient equally + grad_kl_per_row = grad_kl_div / (b * sq) # scalar value for each row # Backward through sum(dim=-1): broadcast back to [b, sq, sk] # Each element in a row contributes to the sum, so gradient is same for all @@ -630,6 +656,7 @@ def forward( mask, sparse_loss, pg_collection, + calculate_per_token_loss, ): """ Fused forward: index_scores never materialized in full. @@ -646,6 +673,7 @@ def forward( mask, sparse_loss, pg_collection, + calculate_per_token_loss, ) # Save for backward (recomputation strategy) @@ -654,6 +682,7 @@ def forward( ctx.loss_coeff = loss_coeff ctx.sparse_loss = sparse_loss ctx.pg_collection = pg_collection + ctx.calculate_per_token_loss = calculate_per_token_loss return topk_indices, loss @@ -677,10 +706,11 @@ def backward(ctx, grad_topk_indices, grad_loss): grad_loss, ctx.pg_collection, causal_mask_override=mask, + calculate_per_token_loss=ctx.calculate_per_token_loss, ) # query and key are detached in forward, so return None for their gradients - return grad_q, grad_weights, grad_k, None, None, None, None, None, None, None, None + return grad_q, grad_weights, grad_k, None, None, None, None, None, None, None, None, None class DSAIndexerLossAutoScaler(torch.autograd.Function): @@ -1201,6 +1231,7 @@ def forward( float_mask, getattr(self.config, "dsa_indexer_use_sparse_loss", False), self.indexer.pg_collection, + self.config.calculate_per_token_loss, ) # Save indexer loss for logging if indexer_loss_coeff > 0: diff --git a/megatron/core/transformer/experimental_attention_variant/dsa_kernels.py b/megatron/core/transformer/experimental_attention_variant/dsa_kernels.py new file mode 100644 index 00000000000..adbcc6e03db --- /dev/null +++ b/megatron/core/transformer/experimental_attention_variant/dsa_kernels.py @@ -0,0 +1,1030 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +""" +DSA kernel wrappers for Megatron's DSv4 sparse attention. + +Mirrors the three integration paths of the old standalone ``dsa_kernels`` +package, but built on top of + +* :mod:`cudnn.deepseek_sparse_attention` (a.k.a. ``DSA``) — CuTe-DSL backward + + indexer score kernels + TRT-LLM radix top-K, shipped as part of + cuDNN Frontend. +* :mod:`flash_mla` — production sparse-attention forward kernel, expected to + be available as a separate PyPI package. + +Public API (same shape as the old ``dsa_kernels`` package): + +* ``build_flat_topk_idxs`` / ``local_to_global_flat`` — index helpers. +* ``dsa_sparse_attn`` — Path A / Path C step 2, differentiable sparse attention. +* ``indexer_topk`` — Path C inference indexer scoring + top-K. +* ``fused_indexer_sparse_attn`` — Path B training, fused indexer loss + + sparse attention with shared backward. +""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch +from torch import Tensor + +# --------------------------------------------------------------------------- +# Lazy kernel imports +# --------------------------------------------------------------------------- + + +_flash_mla_sparse_fwd = None +_DSA = None + + +def _ensure_flash_mla(): + """Lazily import the FlashMLA sparse-forward kernel. + + FlashMLA ships ``flash_mla_sparse_fwd`` with a multi-head-KV signature; + :func:`_dsa_fwd_flash_mla` below is a thin adapter that unbatches the + DSA-shape inputs and pads ``TopK`` to the alignment expected by + FlashMLA's SM90 / SM100 kernels. + """ + global _flash_mla_sparse_fwd + if _flash_mla_sparse_fwd is not None: + return + + try: + from flash_mla import flash_mla_sparse_fwd as _fwd + except ImportError as e: + raise ImportError( + "FlashMLA is required for DSA sparse attention forward. " + "Install from https://github.com/deepseek-ai/FlashMLA/tree/nv_dev " + "so that `from flash_mla import flash_mla_sparse_fwd` succeeds." + ) from e + _flash_mla_sparse_fwd = _fwd + + +def _get_topk_alignment() -> int: + """Minimum ``TopK`` alignment required by the current GPU architecture. + + * SM90 : dual-warpgroup loop steps by 2 blocks → ``2 * B_TOPK = 128`` + * SM100: single-pipeline loop steps by 1 block → ``B_TOPK`` (64 for + head64, 128 for head128). DSA uses ``D = 512`` which maps to the + head64 kernel path → 64. + """ + sm = torch.cuda.get_device_capability() + if sm[0] >= 10: + return 64 + return 128 + + +def _dsa_fwd_flash_mla( + q: Tensor, + kv: Tensor, + topk_idxs: Tensor, + softmax_scale: float, + d_v: int = 512, + attn_sink: Optional[Tensor] = None, + topk_length: Optional[Tensor] = None, + indexer_topk: int = 0, +) -> Tuple[Tensor, Tensor, Optional[Tensor]]: + """DSA-shaped adapter around :func:`flash_mla.flash_mla_sparse_fwd`. + + Accepts flat (unbatched) tensors with global indices; pads ``TopK`` to + the GPU-specific alignment; returns ``(out, lse, lse_indexer)``. + """ + assert not ( + indexer_topk > 0 and topk_length is not None + ), "indexer_topk > 0 requires non-compact mode (topk_length must be None)" + _ensure_flash_mla() + + _total_S_q, _H, _D = q.shape + TopK = topk_idxs.shape[-1] + topk_align = _get_topk_alignment() + TopK_padded = (TopK + topk_align - 1) // topk_align * topk_align + if TopK_padded != TopK: + pad_width = TopK_padded - TopK + topk_idxs = torch.nn.functional.pad(topk_idxs, (0, pad_width), value=-1) + + kv_3d = kv.unsqueeze(1) # (total_S_kv, 1, D) h_kv=1 + indices = topk_idxs.unsqueeze(1) # (total_S_q, 1, TopK_padded) h_kv=1 + + with torch.cuda.nvtx.range("flash_mla_sparse_fwd"): + res = _flash_mla_sparse_fwd( + q, + kv_3d, + indices, + softmax_scale, + d_v=d_v, + attn_sink=attn_sink, + topk_length=topk_length, + indexer_topk=indexer_topk, + ) + if indexer_topk > 0: + out, _max_logits, lse, lse_indexer = res + else: + out, _max_logits, lse = res + lse_indexer = None + + if indexer_topk > 0: + # When indexer_topk == total TopK, lse_indexer should equal lse but + # the kernel may not snapshot correctly; fall back to lse. + if indexer_topk >= TopK: + return out, lse, lse.clone() + return out, lse, lse_indexer + return out, lse, None + + +def _ensure_dsa_namespace(): + """Lazily import the cudnn-frontend DSA namespace.""" + global _DSA + if _DSA is not None: + return + try: + from cudnn import DSA as _ns + except ImportError as e: + raise ImportError( + "cudnn-frontend DSA namespace not available. Install with " + "`pip install nvidia-cudnn-frontend[cutedsl]`." + ) from e + _DSA = _ns + + +# --------------------------------------------------------------------------- +# Index helpers +# --------------------------------------------------------------------------- + + +def local_to_global_flat(local_idxs: Tensor, batch_size: int, seqlen_kv: int) -> Tensor: + """Convert local per-batch indices to global flat indices. + + Follows the convention used by FlashMLA / SparseAttentionBackward: + flat row order is SBHD ``row[s * B + b]``; global index is + ``local * B + b`` for valid entries and ``-1`` otherwise. + + Args: + local_idxs: ``(b, sq, topk)`` int, values in ``[0, seqlen_kv)`` or -1. + batch_size: ``B``. + seqlen_kv: KV sequence length per batch (used for shape assertions + only; callers compute the values). + + Returns: + ``(sq*b, topk)`` int32. + """ + b, sq, topk = local_idxs.shape + assert b == batch_size + + idxs_sb = local_idxs.permute(1, 0, 2).reshape(sq * b, topk) + valid = idxs_sb >= 0 + batch_ids = torch.arange(sq * b, device=local_idxs.device) % b + batch_ids_exp = batch_ids.unsqueeze(1).expand_as(idxs_sb) + idxs_sb = torch.where(valid, idxs_sb * b + batch_ids_exp, idxs_sb) + return idxs_sb.int() + + +def build_flat_topk_idxs( + *idx_groups: Tensor, batch_size: int, seqlen_kv: int, compact: bool = False +) -> Tuple[Tensor, Optional[Tensor]]: + """Combine local per-batch index groups and convert to flat global form. + + Each *idx_group* is ``(b, sq, topk_i)`` with local per-batch KV indices + (already in ``kv_full`` index space, i.e. with any compressed-position + offset applied). ``-1`` marks invalid positions. + + Args: + *idx_groups: one or more ``(b, sq, topk_i)`` int tensors. + batch_size: ``B``. + seqlen_kv: total KV sequence length per batch. + compact: if True, pack valid entries to the front of each row and + additionally return ``topk_length``; if False, leave as-is and + return ``None``. + + Returns: + ``(topk_idxs, topk_length)`` where + ``topk_idxs`` is ``(sq*b, total_topk)`` int32 (flat global) and + ``topk_length`` is ``(sq*b,)`` int32 when ``compact``, else ``None``. + """ + combined = torch.cat(idx_groups, dim=-1) # (b, sq, total_topk) + b, sq, total_topk = combined.shape + + # Globalize first, compact second. Both ops are element-wise + (-1)-preserving, + # so swapping the order is a no-op for correctness; the win is that the + # global indices come out already in (sq*b, total_topk) flat layout, which is + # exactly the row order the cuDNN compactify kernel returns its per-row + # ``length`` in — no extra permute on the length tensor. + global_idxs = local_to_global_flat(combined, b, seqlen_kv) + + topk_length_flat = None + if compact: + if global_idxs.is_cuda: + # Fast path: single warp-per-row CuTe DSL kernel from cuDNN's DSA + # namespace. Replaces a stable argsort + gather + sum + permute + # chain with one global-load + global-store per element. + _ensure_dsa_namespace() + res = _DSA.compactify_wrapper(global_idxs) + global_idxs, topk_length_flat = res["indices"], res["topk_length"] + else: + # CPU fallback so the unit tests that exercise this helper without + # CUDA still work. Production callers always go through the CUDA + # path above. + valid_mask = global_idxs >= 0 + sorted_indices = valid_mask.int().argsort(dim=-1, descending=True, stable=True) + global_idxs = global_idxs.gather(-1, sorted_indices) + topk_length_flat = valid_mask.sum(dim=-1).int() + + return global_idxs, topk_length_flat + + +# --------------------------------------------------------------------------- +# Path A + Path C step 2: differentiable sparse attention +# --------------------------------------------------------------------------- + + +class SparseAttnFunc(torch.autograd.Function): + """SM100 sparse attention fwd + bwd on flat tensors. + + Forward uses :mod:`flash_mla`; backward uses cuDNN Frontend's + :attr:`cudnn.DSA.sparse_attention_backward_wrapper`. + """ + + @staticmethod + def forward( + ctx, + q: Tensor, # (total_sq, H, D) bf16 + kv: Tensor, # (total_skv, D) bf16 + attn_sink: Tensor, # (H,) f32 + topk_idxs: Tensor, # (total_sq, TopK) int32 global + topk_length: Optional[Tensor], # (total_sq,) int32 or None + softmax_scale: float, + indexer_topk: int, + ) -> Tuple[Tensor, Tensor, Optional[Tensor]]: + """Run FlashMLA sparse-attention forward and save tensors for backward.""" + out, lse, lse_indexer = _dsa_fwd_flash_mla( + q, + kv, + topk_idxs, + softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + indexer_topk=indexer_topk, + ) + + ctx.save_for_backward(q, kv, attn_sink, topk_idxs, out, lse) + ctx.softmax_scale = softmax_scale + ctx.topk_length = topk_length + return out, lse, lse_indexer + + @staticmethod + def backward(ctx, dO, d_lse, d_lse_indexer): + """Compute sparse-attention backward via cuDNN DSA wrapper.""" + _ensure_dsa_namespace() + + q, kv, attn_sink, topk_idxs, out, lse = ctx.saved_tensors + + result = _DSA.sparse_attention_backward_wrapper( + q, + kv, + out, + dO, + lse, + attn_sink, + topk_idxs, + softmax_scale=ctx.softmax_scale, + topk_length=ctx.topk_length, + ) + dq, dkv, d_sink = result["dq"], result["dkv"], result["d_sink"] + return dq, dkv, d_sink, None, None, None, None + + +def dsa_sparse_attn( + query: Tensor, + kv: Tensor, + attn_sink: Tensor, + topk_idxs: Tensor, + softmax_scale: float, + topk_length: Optional[Tensor] = None, + indexer_topk: int = 0, +) -> Tensor: + """Sparse attention (Path A / Path C step 2). + + Args: + query: ``(sq, b, np, d)`` bf16 SBHD. + kv: ``(skv, b, d)`` bf16 SBD (K=V). + attn_sink: ``(np,)`` f32. + topk_idxs: ``(sq*b, topk)`` int32 — **flat global** indices produced + by :func:`build_flat_topk_idxs`. + softmax_scale: scalar float. + topk_length: ``(sq*b,)`` int32 — optional compact fast-path. Must be + ``None`` when ``indexer_topk > 0`` (FlashMLA constraint). + indexer_topk: int; ``0`` for Paths A/C, positive for Path B to enable + FlashMLA's ``lse_indexer`` output. + + Returns: + ``(sq, b, np * d_v)`` bf16 output. + """ + sq, b, np_, d = query.shape + skv = kv.shape[0] + + q_flat = query.reshape(sq * b, np_, d) + kv_flat = kv.reshape(skv * b, d) + + out_flat, _lse, _lse_indexer = SparseAttnFunc.apply( + q_flat, kv_flat, attn_sink, topk_idxs, topk_length, softmax_scale, indexer_topk + ) + + d_v = out_flat.shape[-1] + return out_flat.reshape(sq, b, np_, d_v).reshape(sq, b, np_ * d_v) + + +# --------------------------------------------------------------------------- +# Path C inference: indexer scoring + top-K +# --------------------------------------------------------------------------- + + +def _indexer_topk_bshd( + q_bshd: Tensor, k_bsd: Tensor, w_bsh: Tensor, topk: int, ratio: int = 4 +) -> Tuple[Tensor, Tensor, Tensor]: + """BSHD-layout core for :func:`indexer_topk`. + + Internal entry point used by both the public SBHD wrapper and Path B's + ``FusedIndexerSparseAttnFunc.forward`` so the SBHD→BSHD permute can be + performed once at the call site and reused across both the indexer + forward and the score-backward kernels (predict / target). + + Args: + q_bshd: ``(b, sq, idx_nh, idx_hd)`` bf16, C-contiguous. + k_bsd: ``(b, sk, idx_hd)`` bf16, C-contiguous. + w_bsh: ``(b, sq, idx_nh)`` bf16, C-contiguous, **already + ``indexer_softmax_scale``-scaled** by the caller. + topk: number of top-K indices to return per query. + ratio: compression ratio for the kernel's causal mask. + + Returns: + ``(topk_indices, topk_length, scores)`` where: + + * ``topk_indices``: ``(b, sq, topk)`` int32, invalid slots ``-1``. + * ``topk_length``: ``(b, sq)`` int32, per-row valid count. + * ``scores``: ``(b, sq, sk)`` fp32, raw scores from + :attr:`cudnn.DSA.indexer_forward_wrapper` with ``-inf`` on + causally-masked positions. + """ + _ensure_dsa_namespace() + + b, sq, _idx_nh, _idx_hd = q_bshd.shape + sk = k_bsd.shape[1] + device = q_bshd.device + + k_bshd = k_bsd.unsqueeze(2) # (b, sk, 1, idx_hd) + + scores = _DSA.indexer_forward_wrapper(q_bshd, k_bshd, w_bsh, ratio=ratio)[ + "scores" + ] # (b, sq, sk) fp32, -inf on masked positions + + # Top-K selection via the TRT-LLM CuTe-DSL radix kernel. + n_rows = b * sq + scores_flat = scores.reshape(n_rows, sk).contiguous() + q_idx = torch.arange(sq, device=device) + valid_per_q = ((q_idx + 1) // ratio).clamp(max=sk).to(torch.int32) # (sq,) + seq_lens = valid_per_q.repeat(b) # (b*sq,), row-major over (b, sq) + + topk_k = min(topk, sk) + tk_result = _DSA.indexer_top_k_wrapper( + scores_flat, seq_lens, top_k=topk_k, next_n=1, return_val=False + ) + topk_indices = tk_result["indices"].view(b, sq, topk_k) + + if topk_k < topk: + pad = torch.full((b, sq, topk - topk_k), -1, dtype=torch.int32, device=device) + topk_indices = torch.cat([topk_indices, pad], dim=-1) + + topk_length = (topk_indices >= 0).sum(dim=-1).int() # (b, sq) + return topk_indices.int(), topk_length, scores + + +def _sbhd_to_bshd_indexer_inputs( + q_indexer: Tensor, k_indexer: Tensor, weights: Tensor, indexer_softmax_scale: float +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Permute the indexer inputs SBHD→BSHD once, returning both the raw + BSHD weights and (when needed) a separate scaled copy. + + The ``relu(c·x) = c·relu(x)`` trick lets us push the indexer softmax + scale onto ``W`` (``(B, S_q, H)``, small) instead of the score tensor + (``(B, S_q, S_k)``, big). The raw ``w_bsh`` is preserved for the + backward GEMM path, which takes ``sm_scale`` directly. When + ``indexer_softmax_scale == 1.0`` the two views alias each other. + + Returns ``(q_bshd, k_bsd, w_bsh, w_bsh_scaled)``. + """ + q_bshd = q_indexer.permute(1, 0, 2, 3).contiguous() + k_bsd = k_indexer.permute(1, 0, 2).contiguous() + w_bsh = weights.permute(1, 0, 2).contiguous() + + if indexer_softmax_scale != 1.0: + w_bsh_scaled = (w_bsh.float() * indexer_softmax_scale).to(w_bsh.dtype) + else: + w_bsh_scaled = w_bsh + + return q_bshd, k_bsd, w_bsh, w_bsh_scaled + + +def indexer_topk( + q_indexer: Tensor, + k_indexer: Tensor, + weights: Tensor, + topk: int, + ratio: int = 4, + indexer_softmax_scale: float = 1.0, +) -> Tuple[Tensor, Tensor]: + """Score + top-K selection for inference (no KL loss, no backward). + + Built on cuDNN Frontend's CuTe-DSL indexer forward kernel followed by + TRT-LLM's radix top-K kernel. + + Args: + q_indexer: ``(sq, b, idx_nh, idx_hd)`` bf16 SBHD. + k_indexer: ``(sk, b, idx_hd)`` bf16 SBD. + weights: ``(sq, b, idx_nh)`` bf16 SBH — raw (unscaled) weights. + topk: number of top-K indices to select. + ratio: compression ratio for the causal mask. + indexer_softmax_scale: scale applied to the indexer ``Q @ K^T`` + scores (typically ``idx_hd ** -0.5``). Applied internally via + the weights-scaling trick (``relu(c·x) = c·relu(x)`` for + ``c > 0``) so the caller passes raw weights. Default ``1.0`` + means weights are treated as already-scaled. + + Returns: + topk_indices: ``(b, sq, topk)`` int32 — local per-batch indices into + ``k_indexer``; invalid positions are ``-1``. + topk_length: ``(b, sq)`` int32 — per-query valid count. + """ + q_bshd, k_bsd, _w_bsh_raw, w_bsh_scaled = _sbhd_to_bshd_indexer_inputs( + q_indexer, k_indexer, weights, indexer_softmax_scale + ) + topk_indices, topk_length, _ = _indexer_topk_bshd(q_bshd, k_bsd, w_bsh_scaled, topk, ratio) + return topk_indices, topk_length + + +# --------------------------------------------------------------------------- +# Path B: fused indexer + sparse attention (training) +# --------------------------------------------------------------------------- + + +_CLIP_PROB_MIN = torch.finfo(torch.float32).tiny # kept compatible w/ cudnn kernel + + +def _compute_indexer_predict( + q_indexer_bshd: Tensor, + k_indexer_bsd: Tensor, + weights_bsh: Tensor, + topk_indices: Tensor, + qhead_per_kv_head: int, +) -> Tensor: + """Compute ``predict`` distribution (softmax over top-K of indexer scores). + + Wraps :attr:`cudnn.DSA.sparse_indexer_score_recompute_wrapper`. + + Args: + q_indexer_bshd: ``(B, S_q, H_q, D)`` bf16. + k_indexer_bsd: ``(B, S_k, D)`` bf16. + weights_bsh: ``(B, S_q, H_q)`` bf16. + topk_indices: ``(B, S_q, topk)`` int32. + qhead_per_kv_head: ``H_q`` (MQA). + + Returns: + predict: ``(B, S_q, topk)`` fp32, softmax over the top-K axis. + """ + _ensure_dsa_namespace() + result = _DSA.sparse_indexer_score_recompute_wrapper( + q_indexer_bshd, + k_indexer_bsd, + weights_bsh, + topk_indices, + qhead_per_kv_head=qhead_per_kv_head, + ) + return result["predict"] + + +def _compute_attn_target( + q_attn_bshd: Tensor, + k_attn_bsd: Tensor, + lse: Tensor, + topk_indices: Tensor, + softmax_scale: float, + qhead_per_kv_head: int, +) -> Tensor: + """Compute ``target`` distribution (L1-normalised head-sum softmax). + + Wraps :attr:`cudnn.DSA.sparse_attn_score_recompute_wrapper`. + + Shapes match :func:`_compute_indexer_predict`; ``lse`` is + ``(B, S_q, H_q)`` FP32 (comes from the attention forward pass). + """ + _ensure_dsa_namespace() + result = _DSA.sparse_attn_score_recompute_wrapper( + q_attn_bshd, + k_attn_bsd, + lse, + topk_indices, + softmax_scale, + qhead_per_kv_head=qhead_per_kv_head, + ) + return result["target"] + + +def _kl_loss_from_target_predict( + target: Tensor, + predict: Tensor, + topk_indices: Tensor, + loss_coeff: float, + calculate_per_token_loss: bool = False, +) -> Tensor: + """KL(target || predict) reduced over ``(B, S_q)`` and scaled by loss_coeff. + + Rows with no valid top-K positions (early query rows with ratio causal + masking) contribute 0 to the loss — the sparse score kernels produce + garbage for those rows, mirroring ``compute_dsa_indexer_loss``'s + ``row_valid`` handling. The default mean is taken over all ``(B, S_q)`` + positions. Per-token-loss mode returns a raw local sum so finalize can + apply the global token divisor. + """ + eps = _CLIP_PROB_MIN + t = target.clamp(min=eps) + p = predict.clamp(min=eps) + kl_per_row = (t * (torch.log(t) - torch.log(p))).sum(dim=-1) # (B, S_q) + + row_valid = (topk_indices >= 0).any(dim=-1) # (B, S_q) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss_coeff * loss + + +# --------------------------------------------------------------------------- +# Dense path (``sparse_loss=False``) — full-KV indexer loss +# --------------------------------------------------------------------------- + + +def _compute_dense_indexer_score( + q_indexer_bshd: Tensor, + k_indexer_bshd: Tensor, + weights_bsh: Tensor, + qhead_per_kv_head: int, + indexer_softmax_scale: float, + ratio: int, +) -> Tuple[Tensor, Tensor]: + """Dense indexer score forward over the full ``S_k`` axis. + + Wraps :attr:`cudnn.DSA.dense_indexer_score_recompute_wrapper`. Returns + ``(out, denom)`` where + + * ``out`` : ``(B, S_q, S_k)`` fp32, the raw head-reduced score + ``S[b,q,k] = indexer_softmax_scale * sum_h ReLU(Q_h · K_k^T) · W_{b,q,h}`` + with the kernel's ``ratio``-causal mask applied to invalid columns. + * ``denom`` : ``(B, S_q)`` fp32, the LSE denom of ``out`` along + ``S_k`` — i.e. ``predict = exp(out - denom[..., None])`` is the + indexer softmax distribution over the full KV. + + Both outputs are forwarded into :func:`_kl_loss_from_dense_scores` + *and* saved for the dense-path backward, where the dense indexer-grad + kernel consumes them directly. + """ + _ensure_dsa_namespace() + result = _DSA.dense_indexer_score_recompute_wrapper( + q_indexer_bshd, + k_indexer_bshd, + weights_bsh, + qhead_per_kv_head=qhead_per_kv_head, + sm_scale=indexer_softmax_scale, + ratio=ratio, + ) + return result["out"], result["denom"] + + +def _compute_dense_attn_score( + q_attn_bshd: Tensor, + k_attn_bshd: Tensor, + lse: Tensor, + qhead_per_kv_head: int, + softmax_scale: float, + ratio: int, +) -> Tuple[Tensor, Tensor]: + """Dense attention score forward over the full ``S_k`` axis. + + Wraps :attr:`cudnn.DSA.dense_attn_score_recompute_wrapper`. Returns + ``(out, denom)`` where + + * ``out`` : ``(B, S_q, S_k)`` fp32, the head-summed unnormalized + attention probability ``S[b,q,k] = sum_h exp(Q_h · K_k^T · scale - LSE[b,q,h])`` + with ``ratio`` causal mask applied. + * ``denom`` : ``(B, S_q)`` fp32, the L1-norm denom ``sum_k S[b,q,:]``. + ``target = out / denom[..., None]`` is the L1-normalized + head-summed attention distribution. + """ + _ensure_dsa_namespace() + result = _DSA.dense_attn_score_recompute_wrapper( + q_attn_bshd, + k_attn_bshd, + lse, + softmax_scale, + qhead_per_kv_head=qhead_per_kv_head, + ratio=ratio, + ) + return result["out"], result["denom"] + + +def _kl_loss_from_dense_scores( + attn_score: Tensor, + attn_l1norm: Tensor, + index_score: Tensor, + index_lse: Tensor, + loss_coeff: float, + calculate_per_token_loss: bool = False, +) -> Tensor: + """KL(target || predict) over the **full** KV axis, averaged over ``(B, S_q)``. + + Derives ``target = attn_score / attn_l1norm`` (L1-normalised, matches + ``compute_dsa_indexer_loss``'s ``attention_scores / sum`` step) and + ``log_predict = index_score - index_lse`` (LSE-normalised log-softmax), + then computes ``KL = sum_k target * (log target - log predict)`` and + scales by ``loss_coeff``. + + Rows where the kernel's ``ratio`` causal mask leaves no valid KV + position have ``attn_l1norm <= 0`` (L1) or ``index_lse == -inf`` + (LSE); those rows contribute 0 to the loss — the same ``row_valid`` + semantics as the reference ``compute_dsa_indexer_loss``. + """ + eps = _CLIP_PROB_MIN + # row_valid: rows with at least one un-masked KV position. + row_valid = (attn_l1norm > eps) & torch.isfinite(index_lse) + + # Safe denoms: replace invalid rows with a finite value so target / + # log-predict don't produce NaN; the row mask zeroes their KL below. + safe_l1 = attn_l1norm.clamp(min=eps) + safe_lse = torch.where(row_valid, index_lse, torch.zeros_like(index_lse)) + + target = attn_score / safe_l1.unsqueeze(-1) + target_clamped = target.clamp(min=eps) + # Per-position validity: the indexer-score kernel emits -inf at + # ratio-masked positions; those contribute 0 to KL by the + # ``0 · log(0/p) = 0`` convention. Without this gate, the eps-clamp + # on target makes the term ``eps · (log eps - (-inf)) = +inf``. + position_valid = torch.isfinite(index_score) + safe_index_score = torch.where(position_valid, index_score, torch.zeros_like(index_score)) + log_predict = safe_index_score - safe_lse.unsqueeze(-1) + + kl_terms = target_clamped * (torch.log(target_clamped) - log_predict) + kl_terms = torch.where(position_valid, kl_terms, torch.zeros_like(kl_terms)) + kl_per_row = kl_terms.sum(dim=-1) # (B, S_q) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss_coeff * loss + + +class FusedIndexerSparseAttnFunc(torch.autograd.Function): + """Path B: fused indexer (+KL loss) + sparse attention in one autograd. + + Differentiable w.r.t. ``query``, ``kv_full``, ``attn_sink``, + ``q_indexer``, ``k_indexer``, ``weights``. + + Two indexer-loss variants, selected by the ``sparse_loss`` argument + (matches ``compute_dsa_indexer_loss`` in the reference ``dsa.py``): + + * **Sparse loss** (``sparse_loss=True``) — KL is computed only over + the top-K KV positions the indexer has selected. + * **Dense loss** (``sparse_loss=False``, the default) — KL is + computed over *all* causally valid KV positions. + + Both variants share the FlashMLA sparse-attention forward + the + cuDNN sparse-attn backward; only the indexer-loss path branches. + """ + + @staticmethod + def forward( + ctx, + # Sparse attn inputs (differentiable) + query: Tensor, # (sq, b, np, d) bf16 + kv_full: Tensor, # (skv, b, d) bf16 + attn_sink: Tensor, # (np,) f32 + # Window indices (not differentiable) + window_idxs: Tensor, # (b, sq, win_topk) int32 + # Indexer inputs (differentiable) + q_indexer: Tensor, # (sq, b, idx_nh, idx_hd) bf16 + k_indexer: Tensor, # (n_comp, b, idx_hd) bf16 + weights: Tensor, # (sq, b, idx_nh) bf16 — raw (unscaled) + # Scalars + indexer_topk: int, + ratio: int, + softmax_scale: float, + indexer_softmax_scale: float, + loss_coeff: float, + sparse_loss: bool, + kv_offset: int, + calculate_per_token_loss: bool, + ) -> Tuple[Tensor, Tensor]: + """Fused forward: indexer scoring, sparse attention, KL loss, and indexer backward.""" + _ensure_dsa_namespace() + + sq, b, np_, d = query.shape + skv = kv_full.shape[0] + n_comp = k_indexer.shape[0] + idx_nh, idx_hd = q_indexer.shape[2], q_indexer.shape[3] + + effective_topk = min(indexer_topk, n_comp) + + # ---- 1. Permute indexer inputs SBHD->BSHD ONCE. ------------------- + q_idx_bshd, k_idx_bsd, w_bsh, w_bsh_scaled = _sbhd_to_bshd_indexer_inputs( + q_indexer, k_indexer, weights, indexer_softmax_scale + ) + + # ---- 2. Indexer scoring + top-K (with scores retained). ------------- + topk_indices_cmp, _, indexer_scores = _indexer_topk_bshd( + q_idx_bshd, k_idx_bsd, w_bsh_scaled, effective_topk, ratio + ) # topk_indices_cmp: (b, sq, effective_topk) int32; indexer_scores: (b, sq, n_comp) fp32 + + # ---- 3. Combine indices (indexer first, then window). -------------- + compress_topk_idxs = torch.where(topk_indices_cmp >= 0, topk_indices_cmp + kv_offset, -1) + combined_local = torch.cat([compress_topk_idxs, window_idxs], dim=-1) + global_idxs = local_to_global_flat(combined_local, b, skv) + + # ---- 4. FlashMLA forward (non-compact, indexer_topk > 0). --------- + q_flat = query.reshape(sq * b, np_, d) + kv_flat = kv_full.reshape(skv * b, d) + out_flat, lse, lse_indexer = _dsa_fwd_flash_mla( + q_flat, + kv_flat, + global_idxs, + softmax_scale, + attn_sink=attn_sink, + topk_length=None, + indexer_topk=effective_topk, + ) + + # ---- 5. Derive predict from indexer_scores, compute target. -------- + # Attention-path tensors (detached — loss is not differentiable through them). + q_attn_bshd = query.detach().permute(1, 0, 2, 3).contiguous() + k_attn_compressed_bsd = kv_full[kv_offset:].detach().permute(1, 0, 2).contiguous() + lse_indexer_bsqh = lse_indexer.reshape(sq, b, np_).permute(1, 0, 2) + + if sparse_loss: + # Derive predict: gather topk scores from indexer_scores → softmax. + safe_indices = topk_indices_cmp.clamp(min=0).long() + gathered_scores = torch.gather(indexer_scores, dim=2, index=safe_indices) + gathered_scores = torch.where( + topk_indices_cmp >= 0, gathered_scores, torch.finfo(torch.float32).min + ) + predict = torch.softmax(gathered_scores, dim=-1) # (b, sq, topk) fp32 + + target = _compute_attn_target( + q_attn_bshd, + k_attn_compressed_bsd, + lse_indexer_bsqh, + topk_indices_cmp, + softmax_scale, + qhead_per_kv_head=np_, + ) + + if loss_coeff > 0: + indexer_loss = _kl_loss_from_target_predict( + target, predict, topk_indices_cmp, loss_coeff, calculate_per_token_loss + ) + else: + indexer_loss = torch.zeros((), device=query.device, dtype=torch.float32) + else: + # Dense: use full indexer_scores directly + logsumexp. + index_score = indexer_scores # (b, sq, n_comp) fp32 + index_lse = torch.logsumexp(indexer_scores, dim=-1) # (b, sq) fp32 + + attn_score, attn_l1norm = _compute_dense_attn_score( + q_attn_bshd, + k_attn_compressed_bsd.unsqueeze(2), + lse_indexer_bsqh, + qhead_per_kv_head=np_, + softmax_scale=softmax_scale, + ratio=ratio, + ) + + if loss_coeff > 0: + indexer_loss = _kl_loss_from_dense_scores( + attn_score, + attn_l1norm, + index_score, + index_lse, + loss_coeff, + calculate_per_token_loss, + ) + else: + indexer_loss = torch.zeros((), device=query.device, dtype=torch.float32) + + # ---- 6. Eagerly compute indexer backward (grad_loss=1). ------------ + # The actual grad_loss scaling is deferred to backward (when + # DSAIndexerLossAutoScaler provides the correct scale). + indexer_loss_coeff = loss_coeff + if calculate_per_token_loss: + indexer_loss_coeff = loss_coeff * (b * sq) + + unit_grad_loss = torch.ones((), device=query.device, dtype=torch.float32) + + if loss_coeff > 0: + if sparse_loss: + attn_score_for_bwd = target.clone() + index_score_for_bwd = predict.clone() + ig = _DSA.indexer_backward_wrapper( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score_for_bwd, + index_score_for_bwd, + topk_indices_cmp, + sm_scale=indexer_softmax_scale, + loss_coeff=indexer_loss_coeff, + grad_loss=unit_grad_loss, + block_I=128, + ) + else: + attn_score_for_bwd = attn_score.clone() + index_score_for_bwd = index_score.clone() + ig = _DSA.dense_indexer_backward_wrapper( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score_for_bwd, + attn_l1norm, + index_score_for_bwd, + index_lse, + sm_scale=indexer_softmax_scale, + loss_coeff=indexer_loss_coeff, + grad_loss=unit_grad_loss, + ratio=ratio, + block_I=128, + ) + # BSHD -> SBHD (match input layout). + precomputed_grad_q_indexer = ig["d_index_q"].permute(1, 0, 2, 3).contiguous() + precomputed_grad_k_indexer = ig["d_index_k"].permute(1, 0, 2).contiguous() + precomputed_grad_weights = ig["d_weights"].permute(1, 0, 2).contiguous() + else: + precomputed_grad_q_indexer = torch.zeros_like(q_indexer) + precomputed_grad_k_indexer = torch.zeros_like(k_indexer) + precomputed_grad_weights = torch.zeros_like(weights) + + # ---- 7. Save context (only sparse-attn bwd tensors + indexer grads). + ctx.save_for_backward( + q_flat, + kv_flat, + attn_sink, + global_idxs, + out_flat, + lse, + precomputed_grad_q_indexer, + precomputed_grad_k_indexer, + precomputed_grad_weights, + ) + ctx.softmax_scale = softmax_scale + ctx.sq = sq + ctx.b = b + ctx.np_ = np_ + ctx.d = d + ctx.skv = skv + + # ---- 8. Return. --------------------------------------------------- + d_v = out_flat.shape[-1] + output = out_flat.reshape(sq, b, np_, d_v).reshape(sq, b, np_ * d_v) + return output, indexer_loss + + @staticmethod + def backward(ctx, grad_output, grad_loss): + """Backward: sparse attention bwd + scale pre-computed indexer grads.""" + ( + q_flat, + kv_flat, + attn_sink, + global_idxs, + out_flat, + lse, + precomputed_grad_q_indexer, + precomputed_grad_k_indexer, + precomputed_grad_weights, + ) = ctx.saved_tensors + + sq, b, np_, d = ctx.sq, ctx.b, ctx.np_, ctx.d + skv = ctx.skv + + # ---- 1. Sparse attn backward. ------------------------------------- + d_v = out_flat.shape[-1] + dO_flat = grad_output.reshape(sq * b, np_, d_v) + + attn_bwd = _DSA.sparse_attention_backward_wrapper( + q_flat, + kv_flat, + out_flat, + dO_flat, + lse, + attn_sink, + global_idxs, + softmax_scale=ctx.softmax_scale, + topk_length=None, + ) + grad_query = attn_bwd["dq"].reshape(sq, b, np_, d) + grad_kv_full = attn_bwd["dkv"].reshape(skv, b, d) + d_sink = attn_bwd["d_sink"] + + # ---- 2. Scale pre-computed indexer grads by grad_loss. ------------- + grad_q_indexer = precomputed_grad_q_indexer * grad_loss + grad_k_indexer = precomputed_grad_k_indexer * grad_loss + grad_weights = precomputed_grad_weights * grad_loss + + # Grads: query, kv_full, attn_sink, window_idxs, q_indexer, k_indexer, + # weights, indexer_topk, ratio, softmax_scale, indexer_softmax_scale, + # loss_coeff, sparse_loss, kv_offset, calculate_per_token_loss + return ( + grad_query, + grad_kv_full, + d_sink, + None, + grad_q_indexer, + grad_k_indexer, + grad_weights, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +def fused_indexer_sparse_attn( + query: Tensor, + kv_full: Tensor, + attn_sink: Tensor, + window_idxs: Tensor, + q_indexer: Tensor, + k_indexer: Tensor, + weights: Tensor, + indexer_topk: int, + ratio: int, + softmax_scale: float, + indexer_softmax_scale: float = 1.0, + loss_coeff: float = 0.0, + sparse_loss: bool = False, + kv_offset: int = 0, + calculate_per_token_loss: bool = False, +) -> Tuple[Tensor, Tensor]: + """Path B (training): fused indexer (+KL loss) + sparse attention. + + See :class:`FusedIndexerSparseAttnFunc` for the detailed data flow. + + Args: + query: ``(sq, b, np, d)`` bf16 SBHD — attention query. + kv_full: ``(skv, b, d)`` bf16 SBD — original + compressed KV. + attn_sink: ``(np,)`` f32 — learnable sink per head. + window_idxs: ``(b, sq, win_topk)`` int32 — local window indices. + q_indexer: ``(sq, b, idx_nh, idx_hd)`` bf16 — indexer query. + k_indexer: ``(n_comp, b, idx_hd)`` bf16 — indexer key (compressed). + weights: ``(sq, b, idx_nh)`` bf16 — raw indexer weights. + indexer_topk: number of top-K compressed positions to select. + ratio: compression ratio used for the causal mask. + softmax_scale: attention ``Q @ K^T`` scale, typically + ``1/sqrt(v_head_dim)``. + indexer_softmax_scale: indexer ``Q @ K^T`` scale, typically + ``1/sqrt(idx_hd)``. Applied internally — caller passes raw + (unscaled) ``weights``. + loss_coeff: coefficient scaling the KL divergence loss. + sparse_loss: if ``True``, KL is computed only over the top-K + positions (cheap, less informative); if ``False`` (the + default, matches ``transformer_config.dsa_indexer_use_sparse_loss``), + KL is computed over the full causally-valid KV (more + informative, matches the DeepSeek-V3.2 paper, larger + intermediate-tensor footprint). See + :class:`FusedIndexerSparseAttnFunc` for the full data flow + of each variant. + kv_offset: start of compressed region within ``kv_full``. + calculate_per_token_loss: if True, report raw local KL sum and + compensate the cuDNN backward wrappers' local averaging. + + Returns: + ``(output, indexer_loss)`` where ``output`` is ``(sq, b, np * d_v)`` + bf16 and ``indexer_loss`` is a scalar f32. + """ + return FusedIndexerSparseAttnFunc.apply( + query, + kv_full, + attn_sink, + window_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk, + ratio, + softmax_scale, + indexer_softmax_scale, + loss_coeff, + sparse_loss, + kv_offset, + calculate_per_token_loss, + ) + + +__all__ = [ + "build_flat_topk_idxs", + "local_to_global_flat", + "dsa_sparse_attn", + "indexer_topk", + "fused_indexer_sparse_attn", +] diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 6c33e56e70f..7d54f22a4c0 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -564,64 +564,6 @@ def _fused_forward( output = paged_stash_group_commit(output, name="grouped_mlp") return output - def bias_act_func(self, intermediate_parallel, bias_parallel, permuted_probs): - """ - Applies bias and activation function to the output of linear_fc1. - """ - if self.config.use_te_activation_func: - if bias_parallel is not None: - intermediate_parallel = intermediate_parallel + bias_parallel - intermediate_parallel = self.activation_func(intermediate_parallel) - if permuted_probs is not None: - original_dtype = intermediate_parallel.dtype - intermediate_parallel = intermediate_parallel * permuted_probs - intermediate_parallel = intermediate_parallel.to(original_dtype) - elif self.config.bias_activation_fusion: - if self.activation_func == F.silu and self.config.gated_linear_unit: - # dtype is handled inside the fused kernel - intermediate_parallel = weighted_bias_swiglu_impl( - intermediate_parallel, - bias_parallel, - permuted_probs, - self.config.activation_func_fp8_input_store, - self.config.activation_func_clamp_value, - ) - elif self.activation_func == quick_gelu and self.config.gated_linear_unit: - intermediate_parallel = weighted_bias_quick_geglu_impl( - intermediate_parallel, - bias_parallel, - permuted_probs, - self.config.activation_func_fp8_input_store, - self.config.glu_linear_offset, - self.config.activation_func_clamp_value, - ) - else: - raise ValueError("Only support fusion of swiglu and quick_gelu in TEGroupedMLP.") - elif self.activation_func == squared_relu and self.config.use_fused_weighted_squared_relu: - assert bias_parallel is None, "Bias is not supported with fused weighted squared relu." - intermediate_parallel = weighted_squared_relu_impl( - intermediate_parallel, permuted_probs - ) - else: - if self.config.gated_linear_unit: - - def glu(x): - x_glu, x_linear = torch.chunk(x, 2, dim=-1) - if (val := self.config.activation_func_clamp_value) is not None: - x_glu = x_glu.clamp(min=None, max=val) - x_linear = x_linear.clamp(min=-val, max=val) - return self.config.activation_func(x_glu) * ( - x_linear + self.config.glu_linear_offset - ) - - intermediate_parallel = glu(intermediate_parallel) - else: - intermediate_parallel = self.activation_func(intermediate_parallel) - original_dtype = intermediate_parallel.dtype - intermediate_parallel = intermediate_parallel * permuted_probs - intermediate_parallel = intermediate_parallel.to(original_dtype) - return intermediate_parallel - def forward( self, permuted_local_hidden_states: torch.Tensor, @@ -775,7 +717,7 @@ def glu(x): self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput() with moe_act_manager as fc1_output: bias_act_output = self.activation_checkpoint.checkpoint( - self.bias_act_func, fc1_output, bias_parallel, permuted_probs + bias_act_func, fc1_output, bias_parallel, permuted_probs ) else: with moe_act_manager as fc1_output: diff --git a/megatron/core/transformer/moe/fused_a2a.py b/megatron/core/transformer/moe/fused_a2a.py index e3d19e88d7f..2cb37e5d235 100644 --- a/megatron/core/transformer/moe/fused_a2a.py +++ b/megatron/core/transformer/moe/fused_a2a.py @@ -52,9 +52,12 @@ def get_buffer(group: torch.distributed.ProcessGroup, hidden_bytes: int): num_nvl_bytes = max( config.get_nvl_buffer_size_hint(hidden_bytes, group.size()), num_nvl_bytes ) - num_rdma_bytes = max( - config.get_rdma_buffer_size_hint(hidden_bytes, group.size()), num_rdma_bytes - ) + # Local-only EP groups do not need an RDMA buffer, and DeepEP builds + # without internode support may not expose RDMA size hints. + if group.size() > torch.cuda.device_count(): + num_rdma_bytes = max( + config.get_rdma_buffer_size_hint(hidden_bytes, group.size()), num_rdma_bytes + ) # Allocate buffer if not existed or not enough buffer # NOTES: the adaptive routing configuration of the network **must be off** @@ -276,6 +279,9 @@ def set_deepep_num_sms(num_sms): _hybrid_ep_buffer = None +# HybridEP dispatch/combine kernels use 64-token chunks for their public APIs. +HYBRIDEP_TOKEN_ALIGNMENT = 64 + def init_hybrid_ep_buffer( group: torch.distributed.ProcessGroup, diff --git a/megatron/core/transformer/moe/token_dispatcher.py b/megatron/core/transformer/moe/token_dispatcher.py index 7f8b6a1c9a0..45aa652d453 100644 --- a/megatron/core/transformer/moe/token_dispatcher.py +++ b/megatron/core/transformer/moe/token_dispatcher.py @@ -18,6 +18,7 @@ ) from megatron.core.transformer.enums import CudaGraphModule from megatron.core.transformer.moe.fused_a2a import ( + HYBRIDEP_TOKEN_ALIGNMENT, fused_combine, fused_dispatch, hybrid_ep_combine, @@ -1049,16 +1050,46 @@ def __init__( self.moe_expert_rank_capacity_factor = self.config.moe_expert_rank_capacity_factor self.over_budget = torch.zeros(1, dtype=torch.bool, device='cuda') + # THD sequence packing can produce different token counts per rank. + # HybridEP dispatch expects equal per-rank input sizes, so metadata and + # hidden states are padded to the group-wide max and trimmed in combine. + self._original_num_tokens: Optional[int] = None + self._padded_num_tokens: Optional[int] = None def setup_metadata(self, routing_map: torch.Tensor, probs: torch.Tensor): num_tokens = routing_map.shape[0] - self.routing_map = routing_map.reshape(num_tokens, self.num_experts) - self.token_probs = probs.reshape(num_tokens, self.num_experts) + self._original_num_tokens = num_tokens + + padded_num_tokens = num_tokens + if self.config.sequence_packing_scheduler is not None: + # Use the actual tp_ep max so all ranks in the MoE communication + # group pass the same token count to HybridEP. + max_num_tokens_across_ep = torch.tensor( + [num_tokens], device=routing_map.device, dtype=torch.long + ) + torch.distributed.all_reduce( + max_num_tokens_across_ep, op=torch.distributed.ReduceOp.MAX, group=self.group + ) + padded_num_tokens = int(max_num_tokens_across_ep.item()) + padded_num_tokens += -padded_num_tokens % HYBRIDEP_TOKEN_ALIGNMENT + self._padded_num_tokens = padded_num_tokens + + routing_map = routing_map.reshape(num_tokens, self.num_experts) + probs = probs.reshape(num_tokens, self.num_experts) + if self.config.sequence_packing_scheduler is not None and padded_num_tokens > num_tokens: + pad_rows = padded_num_tokens - num_tokens + routing_map = torch.cat( + [routing_map, routing_map.new_zeros((pad_rows, self.num_experts))], dim=0 + ) + probs = torch.cat([probs, probs.new_zeros((pad_rows, self.num_experts))], dim=0) + + self.routing_map = routing_map + self.token_probs = probs if self.moe_expert_rank_capacity_factor is not None: pad_multiple = get_align_size_for_quantization(self.config) budget = int( - routing_map.shape[0] + padded_num_tokens * self.config.moe_router_topk * self.moe_expert_rank_capacity_factor ) @@ -1066,7 +1097,7 @@ def setup_metadata(self, routing_map: torch.Tensor, probs: torch.Tensor): self.num_permuted_tokens = budget # Compute the capacity for each expert at the drop_and_pad mode if self.drop_and_pad: - num_out_tokens = num_tokens * self.config.moe_router_topk + num_out_tokens = padded_num_tokens * self.config.moe_router_topk # Drop and pad the input to capacity. self.capacity = get_capacity( num_tokens=num_out_tokens, @@ -1095,6 +1126,11 @@ def dispatch( self.token_probs = self.token_probs.float() # downcast or upcast if self.config.fp8 or self.config.fp4: self.pad_multiple = get_align_size_for_quantization(self.config) + if self._padded_num_tokens is not None and hidden_states.shape[0] < self._padded_num_tokens: + pad_rows = self._padded_num_tokens - hidden_states.shape[0] + hidden_states = torch.cat( + [hidden_states, hidden_states.new_zeros((pad_rows, hidden_states.shape[-1]))], dim=0 + ) dispatched_hidden, self.dispatched_probs, _, tokens_per_expert, self.handle = ( hybrid_ep_dispatch( x=hidden_states, @@ -1137,12 +1173,20 @@ def combine( pad_multiple=self.pad_multiple, fused=self.config.moe_permute_fusion_into_hybridep, ) + if ( + self._padded_num_tokens is not None + and self._original_num_tokens is not None + and hidden_states.shape[0] > self._original_num_tokens + ): + hidden_states = hidden_states[: self._original_num_tokens] # Release the used handle/num_permuted_tokens which could change in each iteration. # For drop_and_pad mode, we don't need to reset the num_permuted_tokens and # num_dispatched_tokens, because their values never change. self.handle = None if not self.drop_and_pad: self.num_permuted_tokens = None + self._original_num_tokens = None + self._padded_num_tokens = None return hidden_states def get_permuted_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index b85f157e3da..ffe15867b7c 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -337,6 +337,11 @@ class TransformerConfig(ModelParallelConfig): """Whether to use dense mode for compressed sparse attention. If True, the CSA indexer will be disabled.""" + apply_dsa_kernel_fusion: bool = False + """If True, use fused DSA sparse-attention kernels (FlashMLA forward + cuDNN DSA backward, + indexer scoring, and top-K selection). Requires ``flash_mla`` and ``nvidia-cudnn-frontend`` + with CuTe-DSL support. When False, falls back to unfused PyTorch implementations.""" + #################### # linear attention #################### @@ -365,6 +370,9 @@ class TransformerConfig(ModelParallelConfig): linear_num_value_heads: Optional[int] = 32 """Number of value and gate heads for the gated delta net.""" + pre_gated_delta_rule_impl: Literal["unfused", "fused_streamed", "fused_mega"] = "unfused" + """Pre-gated-delta-rule implementation for GatedDeltaNet.""" + #################### # initialization #################### @@ -1385,6 +1393,21 @@ def __post_init__(self): self.experimental_attention_variant = self.linear_attention_type self.linear_attention_type = None + valid_pre_gdr_impls = ("unfused", "fused_streamed", "fused_mega") + if self.pre_gated_delta_rule_impl not in valid_pre_gdr_impls: + raise ValueError( + "pre_gated_delta_rule_impl must be one of " + f"{valid_pre_gdr_impls}, got {self.pre_gated_delta_rule_impl!r}." + ) + if ( + self.pre_gated_delta_rule_impl != "unfused" + and self.experimental_attention_variant != "gated_delta_net" + ): + raise ValueError( + "pre_gated_delta_rule_impl can select a fused path only when " + "experimental_attention_variant='gated_delta_net'." + ) + if self.experimental_attention_variant in ["gated_delta_net"]: assert ( self.linear_attention_freq is not None @@ -1442,6 +1465,44 @@ def __post_init__(self): assert not self.qk_clip, "QK clipping is not supported with DSv4 Hybrid Attention." self.hetereogenous_dist_checkpoint = True + if self.apply_dsa_kernel_fusion: + assert ( + torch.cuda.is_available() + ), "apply_dsa_kernel_fusion requires a CUDA device, but none is available." + sm = torch.cuda.get_device_capability() + assert sm[0] >= 10, ( + f"apply_dsa_kernel_fusion requires SM100+ (Blackwell or later), " + f"but current device has compute capability {sm[0]}.{sm[1]}." + ) + + _flash_mla_available = True + try: + from flash_mla import flash_mla_sparse_fwd # noqa: F401 + except ImportError: + _flash_mla_available = False + + _cudnn_dsa_available = True + try: + from cudnn import DSA # noqa: F401 + except ImportError: + _cudnn_dsa_available = False + + if not _flash_mla_available or not _cudnn_dsa_available: + missing = [] + if not _flash_mla_available: + missing.append( + "flash_mla (install from " + "https://github.com/deepseek-ai/FlashMLA/tree/nv_dev)" + ) + if not _cudnn_dsa_available: + missing.append("cudnn-frontend DSA (nvidia-cudnn-frontend[cutedsl])") + raise ValueError( + f"apply_dsa_kernel_fusion requires fused DSA kernels, but the " + f"following packages are not available: {', '.join(missing)}. " + f"Install them or pass --no-dsa-kernel-fusion to use the unfused " + f"PyTorch fallback." + ) + if self.fp8: # cannot support first last layer bf16 with delayed scaling if self.first_last_layers_bf16 and self.fp8_recipe == Fp8Recipe.delayed: @@ -2910,10 +2971,9 @@ def _scope_to_str(s): # Needed for passing variable sequences between pp stages. self.variable_seq_lengths = True - # TODO(tailaim): add support for other dispatcher types - assert self.moe_token_dispatcher_type == "alltoall", ( - f"sequence_packing only supports moe_token_dispatcher_type='alltoall', " - f"got '{self.moe_token_dispatcher_type}'" + assert self.moe_token_dispatcher_type in ("alltoall", "flex"), ( + f"sequence_packing only supports moe_token_dispatcher_type in " + f"('alltoall', 'flex'), got '{self.moe_token_dispatcher_type}'" ) supported_schedulers = ['dp_balanced', 'default_dynamic_cp'] @@ -3004,7 +3064,12 @@ class MLATransformerConfig(TransformerConfig): def __post_init__(self): super().__post_init__() - if self.multi_latent_attention and self.apply_rope_fusion and self.rope_type != "yarn": + if ( + self.multi_latent_attention + and self.apply_rope_fusion + and self.rope_type != "yarn" + and self.experimental_attention_variant != "dsv4_hybrid" + ): raise ValueError("apply_rope_fusion for MLA only works with YARN RoPE.") if self.attention_output_gate: diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index d2e090de232..353979f97f9 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -716,9 +716,15 @@ def forward(self, *args, **kwargs): """ # Injected by __call__ for cuda graph keying; not a real forward arg. kwargs.pop("dynamic_inference_decode_only", None) - assert ( - not self.config.enable_hyper_connections - ), "Please use HyperConnectionTransformerLayer instead" + called_from_hybrid_mhc_wrapper = kwargs.pop("_called_from_hybrid_mhc_wrapper", False) + if self.config.enable_hyper_connections and not called_from_hybrid_mhc_wrapper: + raise RuntimeError( + "TransformerLayer.forward() must not be called directly when " + "enable_hyper_connections=True. Use HyperConnectionTransformerLayer " + "for transformer-only stacks; HyperConnectionHybridLayer drives the " + "wrapped TransformerLayer through this path automatically for hybrid " + "stacks." + ) hidden_states, context = self._forward_attention(*args, **kwargs) output = self._forward_mlp( hidden_states, @@ -1604,6 +1610,7 @@ def _get_submodules_under_cudagraphs(self): def forward(self, *args, **kwargs): """Forward pass with MHC recompute manager support.""" kwargs.pop("dynamic_inference_decode_only", None) + kwargs.pop("_called_from_hybrid_mhc_wrapper", None) mhc_recompute_manager = getattr(self, '_mhc_recompute_manager', None) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 853973b92cd..c144deecb69 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1274,6 +1274,12 @@ def validate_args(args, defaults={}): args.fsdp_manual_registration = True warn_rank_0('FSDP manual registration is enabled by default when nccl-ub is enabled') + if args.init_model_with_meta_device and args.data_parallel_sharding_strategy == "no_shard": + raise ValueError( + "Meta device initialization (init_model_with_meta_device=True) is not " + "supported or necessary for the 'no_shard' / 0 sharding strategy." + ) + if args.fsdp_manual_registration: assert ( args.use_megatron_fsdp @@ -2566,6 +2572,7 @@ def _add_network_size_args(parser): "persist_layer_norm", "bias_dropout_fusion", "apply_rope_fusion", + "apply_dsa_kernel_fusion", ] transformer_factory = ArgumentGroupFactory(TransformerConfig, exclude=exclude) transformer_group = transformer_factory.build_group(parser, "transformer configuration") @@ -4576,6 +4583,10 @@ def _add_mla_args(parser): def _add_experimental_attention_variant_args(parser): group = parser.add_argument_group(title="experimental_attention_variant") + # NOTE: --pre-gated-delta-rule-impl is auto-generated from the + # TransformerConfig.pre_gated_delta_rule_impl field by ArgumentGroupFactory + # (see _add_transformer_engine_args / build_group), so it must NOT be + # registered manually here — doing so raises an argparse conflict. # Linear attention group.add_argument( '--linear-attention-freq', @@ -4601,6 +4612,13 @@ def _add_experimental_attention_variant_args(parser): 'transformer layer (valid values: 0, 4, 128). ' 'The list length must equal num_layers.', ) + group.add_argument( + '--no-dsa-kernel-fusion', + action='store_false', + help='Disable fused DSA sparse-attention kernels (FlashMLA + cuDNN DSA) ' + 'and fall back to unfused PyTorch implementations.', + dest='apply_dsa_kernel_fusion', + ) return parser diff --git a/megatron/training/datasets/data_samplers.py b/megatron/training/datasets/data_samplers.py index 430bd8b85da..8b14b975aba 100644 --- a/megatron/training/datasets/data_samplers.py +++ b/megatron/training/datasets/data_samplers.py @@ -106,7 +106,7 @@ def close_nvidia_fds(): maybe_worker_init_fn = worker_init_fn if args.num_workers > 0 else None # Torch dataloader. - if args.dynamic_context_parallel: + if args.dynamic_context_parallel or getattr(args, "use_vanilla_collate_fn", False): extra_kwargs = {"collate_fn": lambda x: x} else: extra_kwargs = {} diff --git a/pyproject.toml b/pyproject.toml index 9ccafd4094e..2fb61c6f36c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,7 @@ dev = [ "megatron-energon[av_decode]~=6.0", "av", "flashinfer-python>=0.5.0,<0.7.0", + "nvidia-cudnn-frontend", "wget", "onnxscript", "fastapi~=0.50", # Forcing a little bit more recent version of fastapi to be compatible with pydantic 2.0 diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json index 67d0ff0c729..aebb83bd7db 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json @@ -4,31 +4,31 @@ "end_step": 25, "step_interval": 1, "values": { - "1": 12.59654, - "2": 12.60484, - "3": 12.59797, - "4": 12.5969, - "5": 12.59289, - "6": 12.59265, - "7": 12.58015, - "8": 12.54318, - "9": 12.5105, - "10": 12.49672, - "11": 12.32881, - "12": 12.2994, - "13": 12.23453, - "14": 12.23315, - "15": 11.817, - "16": 11.80129, - "17": 11.76425, - "18": 11.73993, - "19": 11.60889, - "20": 11.50641, - "21": 11.2694, - "22": 11.37967, - "23": 11.28793, - "24": 11.1633, - "25": 10.99893 + "1": 12.59661, + "2": 12.60483, + "3": 12.59815, + "4": 12.59703, + "5": 12.59271, + "6": 12.5926, + "7": 12.57996, + "8": 12.54298, + "9": 12.51056, + "10": 12.49675, + "11": 12.3289, + "12": 12.29947, + "13": 12.23477, + "14": 12.23314, + "15": 11.81705, + "16": 11.80136, + "17": 11.76443, + "18": 11.73997, + "19": 11.60915, + "20": 11.5065, + "21": 11.26958, + "22": 11.37974, + "23": 11.28805, + "24": 11.16345, + "25": 10.99906 } }, "num-zeros": { @@ -36,31 +36,31 @@ "end_step": 25, "step_interval": 1, "values": { - "1": 521038208.0, - "2": 521665504.0, - "3": 520934880.0, - "4": 521227072.0, - "5": 520995648.0, - "6": 521371264.0, - "7": 521420160.0, - "8": 521056704.0, - "9": 521461120.0, - "10": 521178112.0, - "11": 522280544.0, - "12": 521438976.0, - "13": 521475840.0, - "14": 522445184.0, - "15": 521591392.0, - "16": 521415008.0, - "17": 521026752.0, - "18": 521278528.0, - "19": 521155360.0, - "20": 521133952.0, - "21": 522907776.0, - "22": 521589312.0, - "23": 521352864.0, - "24": 521424384.0, - "25": 523543008.0 + "1": 521037888.0, + "2": 521666304.0, + "3": 520935008.0, + "4": 521226784.0, + "5": 520995904.0, + "6": 521373344.0, + "7": 521419904.0, + "8": 521057824.0, + "9": 521461920.0, + "10": 521178048.0, + "11": 522280352.0, + "12": 521439424.0, + "13": 521476608.0, + "14": 522446400.0, + "15": 521592960.0, + "16": 521416256.0, + "17": 521026624.0, + "18": 521278848.0, + "19": 521153408.0, + "20": 521134528.0, + "21": 522908192.0, + "22": 521590080.0, + "23": 521352192.0, + "24": 521425184.0, + "25": 523544480.0 } }, "mem-allocated-bytes": { @@ -100,31 +100,31 @@ "end_step": 25, "step_interval": 1, "values": { - "1": 52729765888.0, - "2": 60518424576.0, - "3": 60519473152.0, - "4": 60519473152.0, - "5": 60519473152.0, - "6": 60519473152.0, - "7": 60519473152.0, - "8": 60519473152.0, - "9": 60519473152.0, - "10": 60519473152.0, - "11": 60519473152.0, - "12": 60519473152.0, - "13": 60519473152.0, - "14": 60519473152.0, - "15": 60519473152.0, - "16": 60519473152.0, - "17": 60519473152.0, - "18": 60519473152.0, - "19": 60519473152.0, - "20": 60519473152.0, - "21": 60519473152.0, - "22": 60519473152.0, - "23": 60519473152.0, - "24": 60519473152.0, - "25": 60519473152.0 + "1": 52730814464.0, + "2": 60518313984.0, + "3": 60519362560.0, + "4": 60519362560.0, + "5": 60519362560.0, + "6": 60519362560.0, + "7": 60519362560.0, + "8": 60519362560.0, + "9": 60519362560.0, + "10": 60519362560.0, + "11": 60519362560.0, + "12": 60519362560.0, + "13": 60519362560.0, + "14": 60519362560.0, + "15": 60519362560.0, + "16": 60519362560.0, + "17": 60519362560.0, + "18": 60519362560.0, + "19": 60519362560.0, + "20": 60519362560.0, + "21": 60519362560.0, + "22": 60519362560.0, + "23": 60519362560.0, + "24": 60519362560.0, + "25": 60519362560.0 } }, "iteration-time": { @@ -133,29 +133,29 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.45736, + "2": 11.72481, "3": "nan", - "4": 0.82262, + "4": 0.82017, "5": "nan", - "6": 0.81264, + "6": 0.79525, "7": "nan", - "8": 0.81353, + "8": 0.79448, "9": "nan", - "10": 0.81267, + "10": 0.79621, "11": "nan", - "12": 0.81216, + "12": 0.79515, "13": "nan", - "14": 0.81378, + "14": 0.7946, "15": "nan", - "16": 0.81183, + "16": 0.79593, "17": "nan", - "18": 0.81133, + "18": 0.79594, "19": "nan", - "20": 0.81001, + "20": 0.79514, "21": "nan", - "22": 0.8112, + "22": 0.79519, "23": "nan", - "24": 0.81072, + "24": 0.7943, "25": "nan" } } diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json index b1c389f8532..aa7b8ee5165 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json @@ -4,30 +4,30 @@ "end_step": 25, "step_interval": 1, "values": { - "1": 12.61164, - "2": 12.60596, - "3": 12.60283, - "4": 12.59697, - "5": 12.59555, + "1": 12.61156, + "2": 12.6059, + "3": 12.6029, + "4": 12.59685, + "5": 12.59572, "6": 12.59771, - "7": 12.58043, - "8": 12.53852, - "9": 12.51223, - "10": 12.49867, - "11": 12.32368, - "12": 12.29429, - "13": 12.23146, - "14": 12.22821, - "15": 11.82219, - "16": 11.80416, - "17": 11.76127, - "18": 11.73709, - "19": 11.61313, - "20": 11.50155, - "21": 11.26472, - "22": 11.37638, - "23": 11.28391, - "24": 11.15655, + "7": 12.58053, + "8": 12.5385, + "9": 12.5123, + "10": 12.49846, + "11": 12.32384, + "12": 12.29418, + "13": 12.23151, + "14": 12.22833, + "15": 11.82248, + "16": 11.80414, + "17": 11.76134, + "18": 11.73724, + "19": 11.6132, + "20": 11.50159, + "21": 11.2649, + "22": 11.37652, + "23": 11.28407, + "24": 11.15662, "25": 10.99872 } }, @@ -36,31 +36,31 @@ "end_step": 25, "step_interval": 1, "values": { - "1": 523049312.0, - "2": 523676640.0, - "3": 522947296.0, - "4": 523241568.0, - "5": 523021536.0, - "6": 523375648.0, - "7": 523434944.0, - "8": 523086432.0, - "9": 523468448.0, - "10": 523196352.0, - "11": 524296800.0, - "12": 523454400.0, - "13": 523497696.0, - "14": 524480800.0, - "15": 523636416.0, - "16": 523466048.0, - "17": 523080416.0, - "18": 523359776.0, - "19": 523209440.0, - "20": 523228640.0, - "21": 524937728.0, - "22": 523660096.0, - "23": 523415296.0, - "24": 523486336.0, - "25": 525638688.0 + "1": 523048608.0, + "2": 523680800.0, + "3": 522947008.0, + "4": 523241248.0, + "5": 523021472.0, + "6": 523374304.0, + "7": 523436960.0, + "8": 523084096.0, + "9": 523469920.0, + "10": 523195584.0, + "11": 524300000.0, + "12": 523454880.0, + "13": 523499296.0, + "14": 524478016.0, + "15": 523636992.0, + "16": 523464160.0, + "17": 523079488.0, + "18": 523362784.0, + "19": 523210592.0, + "20": 523228960.0, + "21": 524938144.0, + "22": 523659552.0, + "23": 523415648.0, + "24": 523485952.0, + "25": 525637760.0 } }, "mem-allocated-bytes": { @@ -133,29 +133,29 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.59096, + "2": 6.64178, "3": "nan", - "4": 0.89026, + "4": 0.90106, "5": "nan", - "6": 0.88523, + "6": 0.89513, "7": "nan", - "8": 0.88482, + "8": 0.93613, "9": "nan", - "10": 0.88377, + "10": 0.87738, "11": "nan", - "12": 0.90678, + "12": 0.875, "13": "nan", - "14": 0.96674, + "14": 0.8772, "15": "nan", - "16": 0.88644, + "16": 0.87966, "17": "nan", - "18": 0.88775, + "18": 0.87706, "19": "nan", - "20": 0.88634, + "20": 0.8843, "21": "nan", - "22": 0.88696, + "22": 0.88333, "23": "nan", - "24": 0.88377, + "24": 0.87985, "25": "nan" } } diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json index 6cfef695086..b196256bbf4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88505, - "2": 10.88299, - "3": 10.88373, - "4": 10.88158, - "5": 10.87447, + "1": 10.88507, + "2": 10.883, + "3": 10.88365, + "4": 10.88156, + "5": 10.87457, "6": 10.87929, - "7": 10.87659, - "8": 10.87678, - "9": 10.88553, - "10": 10.87325, - "11": 10.87288, - "12": 10.86552, + "7": 10.87652, + "8": 10.87676, + "9": 10.88554, + "10": 10.87329, + "11": 10.87289, + "12": 10.86551, "13": 10.87691, "14": 10.86516, - "15": 10.83547, - "16": 10.82924, - "17": 10.84224, - "18": 10.82515, - "19": 10.83501, - "20": 10.73118, - "21": 10.73829, - "22": 10.72352, + "15": 10.83541, + "16": 10.8292, + "17": 10.84229, + "18": 10.82509, + "19": 10.83507, + "20": 10.7312, + "21": 10.73838, + "22": 10.72358, "23": 10.71551, - "24": 10.68505, - "25": 10.67526, - "26": 10.67647, - "27": 10.63301, - "28": 10.57879, - "29": 10.54344, - "30": 10.52049, - "31": 10.51592, - "32": 10.49785, - "33": 10.47045, - "34": 10.4333, - "35": 10.4406, - "36": 10.41441, - "37": 10.37817, - "38": 10.3948, - "39": 10.35342, - "40": 10.34649, - "41": 10.32452, - "42": 10.29736, - "43": 10.28525, - "44": 10.24548, - "45": 10.27432, - "46": 10.22948, - "47": 10.22077, - "48": 10.1752, - "49": 10.17408, - "50": 10.17192 + "24": 10.6851, + "25": 10.67534, + "26": 10.67645, + "27": 10.63315, + "28": 10.57878, + "29": 10.54348, + "30": 10.5206, + "31": 10.51597, + "32": 10.4979, + "33": 10.4705, + "34": 10.43337, + "35": 10.44068, + "36": 10.41442, + "37": 10.3782, + "38": 10.39484, + "39": 10.35349, + "40": 10.34652, + "41": 10.32459, + "42": 10.29739, + "43": 10.28523, + "44": 10.24556, + "45": 10.27435, + "46": 10.22952, + "47": 10.22079, + "48": 10.17527, + "49": 10.17415, + "50": 10.17196 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1751.0, - "2": 1760.0, - "3": 1866.0, - "4": 1773.0, - "5": 1750.0, - "6": 1620.0, + "1": 1775.0, + "2": 1769.0, + "3": 1796.0, + "4": 1862.0, + "5": 1755.0, + "6": 1725.0, "7": 1878.0, - "8": 1793.0, - "9": 1914.0, - "10": 1731.0, - "11": 1733.0, - "12": 1732.0, - "13": 1734.0, - "14": 1929.0, - "15": 1667.0, - "16": 1808.0, - "17": 1813.0, - "18": 1898.0, - "19": 1750.0, - "20": 1775.0, - "21": 1777.0, - "22": 1779.0, - "23": 1784.0, - "24": 1861.0, - "25": 1741.0, + "8": 1765.0, + "9": 1907.0, + "10": 1710.0, + "11": 1785.0, + "12": 1744.0, + "13": 1850.0, + "14": 1919.0, + "15": 1744.0, + "16": 1908.0, + "17": 1801.0, + "18": 1854.0, + "19": 1673.0, + "20": 1724.0, + "21": 1791.0, + "22": 1754.0, + "23": 1792.0, + "24": 1775.0, + "25": 1743.0, "26": 1865.0, - "27": 1810.0, - "28": 1938.0, - "29": 1907.0, - "30": 1825.0, - "31": 1980.0, - "32": 1991.0, - "33": 2055.0, - "34": 2096.0, - "35": 2072.0, - "36": 2003.0, - "37": 2281.0, - "38": 2111.0, - "39": 2282.0, - "40": 2371.0, - "41": 2560.0, - "42": 2170.0, - "43": 2489.0, - "44": 2218.0, - "45": 2694.0, - "46": 2490.0, - "47": 2558.0, - "48": 2755.0, - "49": 2820.0, - "50": 2732.0 + "27": 1819.0, + "28": 1933.0, + "29": 1946.0, + "30": 1947.0, + "31": 1978.0, + "32": 2064.0, + "33": 2020.0, + "34": 2070.0, + "35": 2229.0, + "36": 2061.0, + "37": 2294.0, + "38": 2234.0, + "39": 2251.0, + "40": 2383.0, + "41": 2455.0, + "42": 2179.0, + "43": 2461.0, + "44": 2277.0, + "45": 2651.0, + "46": 2575.0, + "47": 2629.0, + "48": 2650.0, + "49": 2990.0, + "50": 2643.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.89417, - "3": 0.10129, - "4": 0.09179, - "5": 0.09214, - "6": 0.08908, - "7": 0.08801, - "8": 0.08918, - "9": 0.08828, - "10": 0.08796, - "11": 0.09062, - "12": 0.09124, - "13": 0.09165, - "14": 0.09023, - "15": 0.0912, - "16": 0.08944, - "17": 0.08884, - "18": 0.08769, - "19": 0.0885, - "20": 0.08999, - "21": 0.08844, - "22": 0.08852, - "23": 0.08887, - "24": 0.08899, - "25": 0.08796, - "26": 0.08952, - "27": 0.08824, - "28": 0.08877, - "29": 0.09031, - "30": 0.08944, - "31": 0.08905, - "32": 0.08944, - "33": 0.08928, - "34": 0.08939, - "35": 0.09006, - "36": 0.09071, - "37": 0.08942, - "38": 0.08985, - "39": 0.08976, - "40": 0.09014, - "41": 0.09108, - "42": 0.09153, - "43": 0.09121, - "44": 0.09027, - "45": 0.09019, - "46": 0.08957, - "47": 0.09005, - "48": 0.08892, - "49": 0.09069, - "50": 0.09016 + "2": 5.38071, + "3": 0.11389, + "4": 0.07633, + "5": 0.07702, + "6": 0.07724, + "7": 0.07587, + "8": 0.07637, + "9": 0.07627, + "10": 0.07741, + "11": 0.07469, + "12": 0.07561, + "13": 0.0766, + "14": 0.0767, + "15": 0.077, + "16": 0.07742, + "17": 0.0748, + "18": 0.0762, + "19": 0.07618, + "20": 0.07606, + "21": 0.07551, + "22": 0.07561, + "23": 0.07485, + "24": 0.07584, + "25": 0.07481, + "26": 0.07547, + "27": 0.07589, + "28": 0.07562, + "29": 0.07541, + "30": 0.07522, + "31": 0.07496, + "32": 0.07579, + "33": 0.07535, + "34": 0.07631, + "35": 0.07578, + "36": 0.07617, + "37": 0.07689, + "38": 0.07649, + "39": 0.07734, + "40": 0.07754, + "41": 0.07774, + "42": 0.07736, + "43": 0.0766, + "44": 0.07666, + "45": 0.07774, + "46": 0.07717, + "47": 0.07735, + "48": 0.07653, + "49": 0.07671, + "50": 0.07619 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json index 075cd7259e4..7450f1145e8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93343, - "2": 10.92669, - "3": 10.92551, - "4": 10.92532, + "1": 10.93336, + "2": 10.92666, + "3": 10.9255, + "4": 10.92529, "5": 10.93754, - "6": 10.92826, - "7": 10.92387, - "8": 10.92373, - "9": 10.92337, + "6": 10.9282, + "7": 10.9239, + "8": 10.92371, + "9": 10.92334, "10": 10.92739, - "11": 10.9111, - "12": 10.92744, - "13": 10.89422, + "11": 10.91111, + "12": 10.92741, + "13": 10.8942, "14": 10.89282, - "15": 10.8915, - "16": 10.87288, - "17": 10.86943, - "18": 10.86805, - "19": 10.8535, - "20": 10.80094, - "21": 10.78481, - "22": 10.7646, - "23": 10.76759, - "24": 10.75104, - "25": 10.74834, - "26": 10.71952, - "27": 10.68558, - "28": 10.61472, - "29": 10.58963, - "30": 10.55446, - "31": 10.56398, - "32": 10.54721, - "33": 10.51341, - "34": 10.48397, - "35": 10.48648, - "36": 10.46526, - "37": 10.42756, - "38": 10.42669, - "39": 10.39364, - "40": 10.37201, - "41": 10.35126, - "42": 10.34669, - "43": 10.32133, - "44": 10.30202, + "15": 10.89147, + "16": 10.87299, + "17": 10.86945, + "18": 10.86815, + "19": 10.85351, + "20": 10.80099, + "21": 10.7848, + "22": 10.76461, + "23": 10.76766, + "24": 10.75107, + "25": 10.74838, + "26": 10.71962, + "27": 10.68563, + "28": 10.61479, + "29": 10.58967, + "30": 10.55443, + "31": 10.56401, + "32": 10.5473, + "33": 10.51346, + "34": 10.48399, + "35": 10.48652, + "36": 10.4653, + "37": 10.42766, + "38": 10.42672, + "39": 10.39363, + "40": 10.37209, + "41": 10.35129, + "42": 10.34678, + "43": 10.32137, + "44": 10.30203, "45": 10.30545, - "46": 10.26663, - "47": 10.25281, - "48": 10.21608, - "49": 10.20375, - "50": 10.2102 + "46": 10.26667, + "47": 10.25286, + "48": 10.21606, + "49": 10.20378, + "50": 10.21031 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1967.0, - "2": 1681.0, - "3": 1791.0, - "4": 1723.0, - "5": 1710.0, - "6": 1726.0, - "7": 2091.0, - "8": 1799.0, - "9": 1821.0, - "10": 1745.0, - "11": 1701.0, - "12": 1780.0, - "13": 1825.0, - "14": 1958.0, - "15": 1674.0, - "16": 1793.0, - "17": 1823.0, - "18": 1850.0, - "19": 1702.0, - "20": 1636.0, - "21": 1801.0, - "22": 1760.0, - "23": 1782.0, - "24": 1888.0, - "25": 1744.0, - "26": 1797.0, - "27": 1769.0, - "28": 1972.0, - "29": 1927.0, - "30": 1969.0, - "31": 2034.0, - "32": 2050.0, - "33": 2020.0, - "34": 2131.0, - "35": 2040.0, - "36": 2145.0, - "37": 2290.0, - "38": 2234.0, - "39": 2232.0, - "40": 2329.0, - "41": 2441.0, - "42": 2053.0, - "43": 2381.0, - "44": 2323.0, - "45": 2510.0, - "46": 2497.0, - "47": 2456.0, - "48": 2652.0, - "49": 2864.0, - "50": 2585.0 + "1": 1957.0, + "2": 1776.0, + "3": 1717.0, + "4": 1733.0, + "5": 1722.0, + "6": 1746.0, + "7": 2002.0, + "8": 1742.0, + "9": 1845.0, + "10": 1812.0, + "11": 1756.0, + "12": 1686.0, + "13": 1812.0, + "14": 1923.0, + "15": 1687.0, + "16": 1716.0, + "17": 1783.0, + "18": 1842.0, + "19": 1743.0, + "20": 1657.0, + "21": 1818.0, + "22": 1751.0, + "23": 1730.0, + "24": 1859.0, + "25": 1746.0, + "26": 1811.0, + "27": 1731.0, + "28": 1889.0, + "29": 1964.0, + "30": 1951.0, + "31": 2131.0, + "32": 1977.0, + "33": 1959.0, + "34": 2150.0, + "35": 2172.0, + "36": 2077.0, + "37": 2216.0, + "38": 2163.0, + "39": 2182.0, + "40": 2355.0, + "41": 2401.0, + "42": 2022.0, + "43": 2437.0, + "44": 2310.0, + "45": 2449.0, + "46": 2451.0, + "47": 2567.0, + "48": 2642.0, + "49": 2787.0, + "50": 2661.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.89225, - "3": 0.08163, - "4": 0.08422, - "5": 0.08203, - "6": 0.08251, - "7": 0.08165, - "8": 0.08733, - "9": 0.08533, - "10": 0.08163, - "11": 0.08023, - "12": 0.08236, - "13": 0.0813, - "14": 0.08273, - "15": 0.08182, - "16": 0.08156, - "17": 0.08634, - "18": 0.08175, - "19": 0.08144, - "20": 0.08379, - "21": 0.08315, - "22": 0.08162, - "23": 0.08092, - "24": 0.08149, - "25": 0.0819, - "26": 0.08509, - "27": 0.08243, - "28": 0.08027, - "29": 0.08085, - "30": 0.08176, - "31": 0.08133, - "32": 0.08421, - "33": 0.08126, - "34": 0.08115, - "35": 0.08002, - "36": 0.07981, - "37": 0.08035, - "38": 0.08069, - "39": 0.08506, - "40": 0.0867, - "41": 0.08141, - "42": 0.08516, - "43": 0.08312, - "44": 0.0872, - "45": 0.08844, - "46": 0.08728, - "47": 0.08141, - "48": 0.08513, - "49": 0.07969, - "50": 0.08489 + "2": 2.72479, + "3": 0.08564, + "4": 0.08671, + "5": 0.08368, + "6": 0.08402, + "7": 0.0839, + "8": 0.08632, + "9": 0.08273, + "10": 0.08502, + "11": 0.08406, + "12": 0.0835, + "13": 0.08263, + "14": 0.08616, + "15": 0.08241, + "16": 0.08136, + "17": 0.08143, + "18": 0.08214, + "19": 0.08285, + "20": 0.08171, + "21": 0.08809, + "22": 0.0834, + "23": 0.08276, + "24": 0.08247, + "25": 0.08217, + "26": 0.08161, + "27": 0.08202, + "28": 0.08728, + "29": 0.08245, + "30": 0.08269, + "31": 0.08113, + "32": 0.08216, + "33": 0.08183, + "34": 0.08182, + "35": 0.08279, + "36": 0.0842, + "37": 0.08228, + "38": 0.08517, + "39": 0.08217, + "40": 0.08157, + "41": 0.08172, + "42": 0.08247, + "43": 0.08191, + "44": 0.0822, + "45": 0.08131, + "46": 0.08141, + "47": 0.0818, + "48": 0.08176, + "49": 0.08148, + "50": 0.08199 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json index 1b9f7e91548..c0a11371f57 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88433, - "2": 10.88455, + "1": 10.88436, + "2": 10.88459, "3": 10.88309, - "4": 10.88497, - "5": 10.87886, - "6": 10.87416, - "7": 10.88029, - "8": 10.87382, - "9": 10.88138, - "10": 10.88114, - "11": 10.87549, - "12": 10.86262, - "13": 10.87418, - "14": 10.86552, - "15": 10.83746, - "16": 10.83285, - "17": 10.83752, + "4": 10.88494, + "5": 10.87889, + "6": 10.87417, + "7": 10.88025, + "8": 10.87377, + "9": 10.88146, + "10": 10.88113, + "11": 10.87554, + "12": 10.86259, + "13": 10.87412, + "14": 10.86553, + "15": 10.83742, + "16": 10.83275, + "17": 10.83755, "18": 10.82158, - "19": 10.83534, - "20": 10.73039, - "21": 10.73845, - "22": 10.72167, - "23": 10.71282, - "24": 10.68134, - "25": 10.67711, - "26": 10.67859, - "27": 10.63636, - "28": 10.57712, - "29": 10.54434, - "30": 10.52198, + "19": 10.8353, + "20": 10.73041, + "21": 10.73852, + "22": 10.72163, + "23": 10.71275, + "24": 10.68131, + "25": 10.67714, + "26": 10.67861, + "27": 10.63632, + "28": 10.57717, + "29": 10.54437, + "30": 10.52205, "31": 10.51451, - "32": 10.49261, - "33": 10.47084, + "32": 10.49257, + "33": 10.47083, "34": 10.43254, - "35": 10.44114, - "36": 10.41271, + "35": 10.4411, + "36": 10.4127, "37": 10.37829, - "38": 10.39605, - "39": 10.35296, - "40": 10.34738, - "41": 10.32554, - "42": 10.29871, - "43": 10.2881, - "44": 10.24608, - "45": 10.27178, - "46": 10.22622, - "47": 10.22132, + "38": 10.39602, + "39": 10.35301, + "40": 10.34743, + "41": 10.32556, + "42": 10.29877, + "43": 10.28817, + "44": 10.2461, + "45": 10.27179, + "46": 10.22628, + "47": 10.22134, "48": 10.17358, - "49": 10.17262, - "50": 10.17363 + "49": 10.17266, + "50": 10.17368 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1762.0, - "2": 1751.0, - "3": 1680.0, - "4": 1694.0, - "5": 1780.0, - "6": 1620.0, - "7": 1878.0, - "8": 1712.0, - "9": 1784.0, - "10": 1813.0, - "11": 1724.0, - "12": 1774.0, - "13": 1798.0, - "14": 1902.0, - "15": 1579.0, - "16": 1665.0, - "17": 1839.0, - "18": 1751.0, - "19": 1684.0, - "20": 1696.0, - "21": 1813.0, - "22": 1721.0, - "23": 1705.0, - "24": 1795.0, - "25": 1699.0, - "26": 1767.0, - "27": 1747.0, - "28": 1871.0, - "29": 1847.0, - "30": 1813.0, - "31": 2027.0, - "32": 2003.0, - "33": 2032.0, - "34": 2058.0, - "35": 2136.0, - "36": 2084.0, - "37": 2158.0, - "38": 2049.0, - "39": 2259.0, - "40": 2277.0, - "41": 2445.0, - "42": 2144.0, - "43": 2428.0, - "44": 2234.0, - "45": 2590.0, - "46": 2394.0, - "47": 2567.0, - "48": 2671.0, - "49": 2833.0, - "50": 2640.0 + "1": 1711.0, + "2": 1729.0, + "3": 1743.0, + "4": 1717.0, + "5": 1753.0, + "6": 1571.0, + "7": 1945.0, + "8": 1664.0, + "9": 1763.0, + "10": 1756.0, + "11": 1576.0, + "12": 1740.0, + "13": 1769.0, + "14": 1912.0, + "15": 1642.0, + "16": 1696.0, + "17": 1758.0, + "18": 1759.0, + "19": 1606.0, + "20": 1712.0, + "21": 1851.0, + "22": 1755.0, + "23": 1724.0, + "24": 1738.0, + "25": 1784.0, + "26": 1867.0, + "27": 1786.0, + "28": 1854.0, + "29": 1875.0, + "30": 1824.0, + "31": 1920.0, + "32": 1900.0, + "33": 1953.0, + "34": 2163.0, + "35": 2095.0, + "36": 2073.0, + "37": 2097.0, + "38": 2123.0, + "39": 2225.0, + "40": 2302.0, + "41": 2269.0, + "42": 2037.0, + "43": 2399.0, + "44": 2300.0, + "45": 2662.0, + "46": 2412.0, + "47": 2565.0, + "48": 2779.0, + "49": 3046.0, + "50": 2667.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.00957, - "3": 0.1544, - "4": 0.99524, - "5": 0.2564, - "6": 0.75781, - "7": 1.04132, - "8": 1.3185, - "9": 0.67632, - "10": 0.44667, - "11": 0.97555, - "12": 0.37765, - "13": 0.64289, - "14": 0.57222, - "15": 0.50117, - "16": 0.66784, - "17": 0.5169, - "18": 0.83242, - "19": 0.2859, - "20": 0.59278, - "21": 0.69966, - "22": 0.54056, - "23": 0.60898, - "24": 0.43047, - "25": 0.45108, - "26": 0.85123, - "27": 0.20228, - "28": 0.56624, - "29": 0.74604, - "30": 0.16317, - "31": 0.42617, - "32": 0.85886, - "33": 0.5399, - "34": 0.65074, - "35": 0.4537, - "36": 0.50039, - "37": 0.48866, - "38": 0.48904, - "39": 0.62383, - "40": 0.6464, - "41": 0.60573, - "42": 0.51136, - "43": 0.49241, - "44": 0.6534, - "45": 0.59786, - "46": 0.43762, - "47": 0.56581, - "48": 0.73181, - "49": 0.37909, - "50": 0.40553 + "2": 3.77634, + "3": 0.13665, + "4": 0.61999, + "5": 0.4214, + "6": 0.60429, + "7": 0.78, + "8": 0.94888, + "9": 0.61551, + "10": 0.32844, + "11": 0.8974, + "12": 0.36329, + "13": 0.58346, + "14": 0.52438, + "15": 0.47188, + "16": 0.68395, + "17": 0.51022, + "18": 0.74241, + "19": 0.34832, + "20": 0.50375, + "21": 0.65058, + "22": 0.54757, + "23": 0.5773, + "24": 0.24559, + "25": 0.45371, + "26": 0.96572, + "27": 0.22424, + "28": 0.59083, + "29": 0.69124, + "30": 0.13679, + "31": 0.20347, + "32": 1.05575, + "33": 0.47397, + "34": 0.2823, + "35": 0.64122, + "36": 0.47863, + "37": 0.48564, + "38": 0.58538, + "39": 0.57125, + "40": 0.67499, + "41": 0.5342, + "42": 0.4642, + "43": 0.53306, + "44": 0.6227, + "45": 0.54885, + "46": 0.40888, + "47": 0.51891, + "48": 0.72053, + "49": 0.392, + "50": 0.47703 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json index 77bcb705aee..ff6cbe9b518 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88441, - "2": 10.88419, + "1": 10.8844, + "2": 10.88421, "3": 10.88532, - "4": 10.883, - "5": 10.87516, - "6": 10.87715, - "7": 10.87903, - "8": 10.87677, - "9": 10.88519, - "10": 10.87583, - "11": 10.87419, - "12": 10.86421, - "13": 10.87889, - "14": 10.86664, - "15": 10.83749, - "16": 10.8288, - "17": 10.84361, - "18": 10.82683, - "19": 10.83103, - "20": 10.73111, - "21": 10.7373, - "22": 10.72483, - "23": 10.7144, - "24": 10.68881, - "25": 10.67484, - "26": 10.67693, - "27": 10.63664, - "28": 10.58113, - "29": 10.54327, - "30": 10.52105, - "31": 10.51624, - "32": 10.49654, - "33": 10.46949, - "34": 10.4335, - "35": 10.43987, - "36": 10.41254, - "37": 10.37732, - "38": 10.39273, - "39": 10.35185, - "40": 10.34725, - "41": 10.32346, - "42": 10.29729, - "43": 10.28756, - "44": 10.2445, - "45": 10.27205, - "46": 10.22827, - "47": 10.22063, - "48": 10.1729, - "49": 10.17215, - "50": 10.17272 + "4": 10.88299, + "5": 10.87522, + "6": 10.87714, + "7": 10.87895, + "8": 10.87676, + "9": 10.88522, + "10": 10.87584, + "11": 10.87416, + "12": 10.86431, + "13": 10.87891, + "14": 10.86666, + "15": 10.83746, + "16": 10.82879, + "17": 10.84362, + "18": 10.82682, + "19": 10.83099, + "20": 10.73113, + "21": 10.73729, + "22": 10.72486, + "23": 10.71444, + "24": 10.68878, + "25": 10.67488, + "26": 10.67696, + "27": 10.63674, + "28": 10.58117, + "29": 10.54342, + "30": 10.52112, + "31": 10.5163, + "32": 10.49664, + "33": 10.46954, + "34": 10.43354, + "35": 10.43989, + "36": 10.41255, + "37": 10.37739, + "38": 10.3928, + "39": 10.35194, + "40": 10.34731, + "41": 10.32347, + "42": 10.2973, + "43": 10.28759, + "44": 10.24452, + "45": 10.27209, + "46": 10.22837, + "47": 10.22064, + "48": 10.17294, + "49": 10.17222, + "50": 10.17282 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1803.0, - "2": 1714.0, - "3": 1794.0, - "4": 1754.0, - "5": 1773.0, - "6": 1794.0, - "7": 1908.0, - "8": 1678.0, - "9": 1824.0, - "10": 1858.0, - "11": 1671.0, - "12": 1670.0, - "13": 1868.0, - "14": 1906.0, - "15": 1616.0, - "16": 1880.0, - "17": 1819.0, - "18": 1917.0, - "19": 1773.0, - "20": 1697.0, - "21": 1799.0, - "22": 1849.0, - "23": 1717.0, - "24": 1897.0, - "25": 1811.0, - "26": 1900.0, - "27": 1903.0, - "28": 1878.0, - "29": 1940.0, - "30": 1979.0, - "31": 2025.0, - "32": 1990.0, - "33": 2046.0, - "34": 2077.0, - "35": 2096.0, - "36": 2033.0, - "37": 2294.0, - "38": 2168.0, - "39": 2271.0, - "40": 2373.0, - "41": 2358.0, - "42": 2147.0, - "43": 2512.0, - "44": 2288.0, - "45": 2667.0, - "46": 2552.0, - "47": 2551.0, - "48": 2694.0, - "49": 3041.0, - "50": 2690.0 + "1": 1688.0, + "2": 1766.0, + "3": 1808.0, + "4": 1826.0, + "5": 1783.0, + "6": 1743.0, + "7": 1904.0, + "8": 1758.0, + "9": 1843.0, + "10": 1728.0, + "11": 1697.0, + "12": 1697.0, + "13": 1768.0, + "14": 1807.0, + "15": 1693.0, + "16": 1831.0, + "17": 1859.0, + "18": 1937.0, + "19": 1764.0, + "20": 1770.0, + "21": 1870.0, + "22": 1768.0, + "23": 1841.0, + "24": 1846.0, + "25": 1870.0, + "26": 1963.0, + "27": 1850.0, + "28": 1885.0, + "29": 1998.0, + "30": 1911.0, + "31": 2047.0, + "32": 2012.0, + "33": 1984.0, + "34": 2139.0, + "35": 2068.0, + "36": 2111.0, + "37": 2245.0, + "38": 2192.0, + "39": 2194.0, + "40": 2401.0, + "41": 2378.0, + "42": 2092.0, + "43": 2508.0, + "44": 2314.0, + "45": 2604.0, + "46": 2483.0, + "47": 2525.0, + "48": 2690.0, + "49": 3014.0, + "50": 2733.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.06066, - "3": 0.09819, - "4": 0.0912, - "5": 0.09323, - "6": 0.08804, - "7": 0.08797, - "8": 0.08863, - "9": 0.08857, - "10": 0.08786, - "11": 0.08724, - "12": 0.08845, - "13": 0.08807, - "14": 0.08801, - "15": 0.08885, - "16": 0.08834, - "17": 0.08907, - "18": 0.08813, - "19": 0.08905, - "20": 0.08969, - "21": 0.08898, - "22": 0.08912, - "23": 0.0893, - "24": 0.08913, - "25": 0.08928, - "26": 0.09104, - "27": 0.08956, - "28": 0.08856, - "29": 0.08952, - "30": 0.08786, - "31": 0.08897, - "32": 0.09014, - "33": 0.08931, - "34": 0.08859, - "35": 0.08881, - "36": 0.09112, - "37": 0.09237, - "38": 0.08975, - "39": 0.08821, - "40": 0.0877, - "41": 0.08737, - "42": 0.08741, - "43": 0.08677, - "44": 0.08698, - "45": 0.08663, - "46": 0.08642, - "47": 0.0869, - "48": 0.08709, - "49": 0.08645, - "50": 0.08611 + "2": 5.29589, + "3": 0.11615, + "4": 0.07381, + "5": 0.07443, + "6": 0.07487, + "7": 0.07522, + "8": 0.0743, + "9": 0.07537, + "10": 0.07328, + "11": 0.07392, + "12": 0.07422, + "13": 0.07594, + "14": 0.07444, + "15": 0.07292, + "16": 0.07404, + "17": 0.07374, + "18": 0.07297, + "19": 0.07286, + "20": 0.0727, + "21": 0.07446, + "22": 0.07411, + "23": 0.07574, + "24": 0.0751, + "25": 0.07509, + "26": 0.07355, + "27": 0.07302, + "28": 0.07568, + "29": 0.07544, + "30": 0.07507, + "31": 0.07435, + "32": 0.07469, + "33": 0.07474, + "34": 0.07521, + "35": 0.07533, + "36": 0.07393, + "37": 0.07312, + "38": 0.07401, + "39": 0.07536, + "40": 0.07564, + "41": 0.07499, + "42": 0.07626, + "43": 0.077, + "44": 0.07494, + "45": 0.0752, + "46": 0.07546, + "47": 0.07411, + "48": 0.07553, + "49": 0.07462, + "50": 0.07529 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json index 7e18481b88f..8192146a99b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93399, - "2": 10.92476, - "3": 10.92682, - "4": 10.93022, - "5": 10.93698, - "6": 10.92965, + "1": 10.93393, + "2": 10.92467, + "3": 10.92678, + "4": 10.93013, + "5": 10.93701, + "6": 10.9296, "7": 10.92253, - "8": 10.92212, - "9": 10.92457, - "10": 10.92887, - "11": 10.90984, - "12": 10.92854, - "13": 10.89636, - "14": 10.89646, - "15": 10.89151, - "16": 10.87388, + "8": 10.9221, + "9": 10.92459, + "10": 10.92892, + "11": 10.90982, + "12": 10.92853, + "13": 10.89638, + "14": 10.89651, + "15": 10.8915, + "16": 10.87384, "17": 10.8703, - "18": 10.8717, - "19": 10.85327, - "20": 10.80493, - "21": 10.7837, - "22": 10.7641, - "23": 10.77106, - "24": 10.75051, - "25": 10.74642, - "26": 10.72229, - "27": 10.68745, - "28": 10.61503, - "29": 10.59229, - "30": 10.55596, - "31": 10.56461, - "32": 10.54468, - "33": 10.51563, - "34": 10.48265, - "35": 10.48463, - "36": 10.46505, - "37": 10.42806, - "38": 10.42575, - "39": 10.39425, - "40": 10.37402, - "41": 10.35095, - "42": 10.34773, - "43": 10.31923, - "44": 10.29862, - "45": 10.30319, - "46": 10.26407, - "47": 10.25193, - "48": 10.215, - "49": 10.20174, - "50": 10.21031 + "18": 10.87176, + "19": 10.85332, + "20": 10.80507, + "21": 10.78369, + "22": 10.76407, + "23": 10.77113, + "24": 10.75059, + "25": 10.74651, + "26": 10.72238, + "27": 10.68742, + "28": 10.61512, + "29": 10.59231, + "30": 10.55603, + "31": 10.56468, + "32": 10.54471, + "33": 10.51566, + "34": 10.48267, + "35": 10.48467, + "36": 10.46512, + "37": 10.42808, + "38": 10.4258, + "39": 10.39431, + "40": 10.374, + "41": 10.35096, + "42": 10.34771, + "43": 10.3193, + "44": 10.29873, + "45": 10.30323, + "46": 10.26412, + "47": 10.25198, + "48": 10.21503, + "49": 10.2018, + "50": 10.21033 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1881.0, - "2": 1708.0, - "3": 1776.0, - "4": 1728.0, - "5": 1710.0, - "6": 1765.0, - "7": 2082.0, - "8": 1791.0, - "9": 1899.0, - "10": 1705.0, - "11": 1734.0, - "12": 1770.0, - "13": 1899.0, - "14": 1951.0, - "15": 1709.0, - "16": 1899.0, - "17": 1840.0, - "18": 1800.0, - "19": 1708.0, - "20": 1628.0, - "21": 1827.0, - "22": 1708.0, - "23": 1704.0, - "24": 1826.0, - "25": 1723.0, - "26": 1871.0, - "27": 1882.0, - "28": 1943.0, - "29": 2002.0, - "30": 1945.0, - "31": 2070.0, - "32": 1917.0, - "33": 2049.0, - "34": 2073.0, - "35": 2144.0, - "36": 2141.0, - "37": 2352.0, - "38": 2075.0, - "39": 2247.0, - "40": 2338.0, - "41": 2357.0, - "42": 2115.0, - "43": 2414.0, - "44": 2263.0, - "45": 2650.0, - "46": 2439.0, - "47": 2703.0, - "48": 2508.0, - "49": 2809.0, - "50": 2694.0 + "1": 1889.0, + "2": 1680.0, + "3": 1705.0, + "4": 1742.0, + "5": 1810.0, + "6": 1834.0, + "7": 2160.0, + "8": 1761.0, + "9": 1880.0, + "10": 1715.0, + "11": 1802.0, + "12": 1752.0, + "13": 1784.0, + "14": 1878.0, + "15": 1778.0, + "16": 1781.0, + "17": 1861.0, + "18": 1789.0, + "19": 1755.0, + "20": 1780.0, + "21": 1766.0, + "22": 1644.0, + "23": 1724.0, + "24": 1782.0, + "25": 1763.0, + "26": 1759.0, + "27": 1901.0, + "28": 2028.0, + "29": 1972.0, + "30": 1858.0, + "31": 2101.0, + "32": 1974.0, + "33": 1992.0, + "34": 2159.0, + "35": 2103.0, + "36": 2057.0, + "37": 2299.0, + "38": 2093.0, + "39": 2288.0, + "40": 2297.0, + "41": 2378.0, + "42": 2110.0, + "43": 2366.0, + "44": 2316.0, + "45": 2570.0, + "46": 2513.0, + "47": 2644.0, + "48": 2552.0, + "49": 2881.0, + "50": 2617.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.65413, - "3": 0.08085, - "4": 0.08064, - "5": 0.08045, - "6": 0.08256, - "7": 0.08015, - "8": 0.0813, - "9": 0.07903, - "10": 0.08042, - "11": 0.08177, - "12": 0.08108, - "13": 0.07969, - "14": 0.08025, - "15": 0.08016, - "16": 0.08011, - "17": 0.07942, - "18": 0.08033, - "19": 0.07982, - "20": 0.07968, - "21": 0.08004, - "22": 0.08175, - "23": 0.08418, - "24": 0.08754, - "25": 0.07846, - "26": 0.08009, - "27": 0.07834, - "28": 0.08125, - "29": 0.07899, - "30": 0.08056, - "31": 0.07954, - "32": 0.07996, - "33": 0.07917, - "34": 0.07986, - "35": 0.07983, - "36": 0.08227, - "37": 0.08683, - "38": 0.08139, - "39": 0.07946, - "40": 0.07988, - "41": 0.08031, - "42": 0.08136, - "43": 0.08108, - "44": 0.08087, - "45": 0.07824, - "46": 0.0806, - "47": 0.0811, - "48": 0.08044, - "49": 0.07912, - "50": 0.08042 + "2": 2.8133, + "3": 0.08302, + "4": 0.08329, + "5": 0.0886, + "6": 0.08329, + "7": 0.08289, + "8": 0.08287, + "9": 0.08296, + "10": 0.08304, + "11": 0.08418, + "12": 0.0859, + "13": 0.08362, + "14": 0.08383, + "15": 0.08239, + "16": 0.08386, + "17": 0.08287, + "18": 0.08319, + "19": 0.08465, + "20": 0.08561, + "21": 0.08314, + "22": 0.08262, + "23": 0.08335, + "24": 0.08297, + "25": 0.08177, + "26": 0.08156, + "27": 0.0815, + "28": 0.08452, + "29": 0.08458, + "30": 0.08139, + "31": 0.08305, + "32": 0.08222, + "33": 0.08213, + "34": 0.08104, + "35": 0.08202, + "36": 0.08236, + "37": 0.08256, + "38": 0.08221, + "39": 0.08168, + "40": 0.08854, + "41": 0.08156, + "42": 0.08175, + "43": 0.08153, + "44": 0.08187, + "45": 0.08262, + "46": 0.08123, + "47": 0.08219, + "48": 0.08183, + "49": 0.0851, + "50": 0.0945 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json index 5a27b3f8d56..7226a215f6d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88533, - "2": 10.88323, - "3": 10.88389, - "4": 10.88685, + "1": 10.8853, + "2": 10.88321, + "3": 10.88381, + "4": 10.88683, "5": 10.8789, - "6": 10.87638, - "7": 10.8797, - "8": 10.87233, - "9": 10.88488, - "10": 10.87864, - "11": 10.87362, - "12": 10.86361, + "6": 10.87631, + "7": 10.87967, + "8": 10.87231, + "9": 10.88486, + "10": 10.87858, + "11": 10.87356, + "12": 10.86371, "13": 10.87879, - "14": 10.865, - "15": 10.83819, + "14": 10.86496, + "15": 10.83824, "16": 10.83255, - "17": 10.84053, - "18": 10.82214, - "19": 10.83598, - "20": 10.72905, - "21": 10.73896, - "22": 10.72498, - "23": 10.71146, - "24": 10.68267, - "25": 10.67564, - "26": 10.67938, - "27": 10.63466, - "28": 10.57945, - "29": 10.54432, - "30": 10.52102, - "31": 10.5136, - "32": 10.49426, - "33": 10.46836, - "34": 10.43371, - "35": 10.43863, - "36": 10.41292, - "37": 10.37869, - "38": 10.39311, - "39": 10.35118, - "40": 10.34578, - "41": 10.32306, - "42": 10.29734, - "43": 10.28519, - "44": 10.24436, - "45": 10.27149, - "46": 10.22645, - "47": 10.22022, - "48": 10.1732, - "49": 10.17067, - "50": 10.17205 + "17": 10.84051, + "18": 10.82218, + "19": 10.83596, + "20": 10.72909, + "21": 10.73906, + "22": 10.72491, + "23": 10.71153, + "24": 10.68273, + "25": 10.67559, + "26": 10.67939, + "27": 10.63467, + "28": 10.57956, + "29": 10.54433, + "30": 10.52116, + "31": 10.51362, + "32": 10.49421, + "33": 10.46838, + "34": 10.43376, + "35": 10.43862, + "36": 10.41297, + "37": 10.37868, + "38": 10.39314, + "39": 10.35125, + "40": 10.34579, + "41": 10.3231, + "42": 10.29733, + "43": 10.2853, + "44": 10.24443, + "45": 10.27153, + "46": 10.22648, + "47": 10.22033, + "48": 10.17323, + "49": 10.1707, + "50": 10.17214 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1752.0, - "2": 1759.0, - "3": 1797.0, - "4": 1699.0, - "5": 1723.0, - "6": 1652.0, - "7": 1826.0, - "8": 1721.0, - "9": 1738.0, - "10": 1755.0, - "11": 1713.0, - "12": 1633.0, - "13": 1838.0, - "14": 1805.0, - "15": 1624.0, - "16": 1772.0, - "17": 1803.0, - "18": 1885.0, - "19": 1669.0, - "20": 1740.0, - "21": 1750.0, - "22": 1711.0, - "23": 1656.0, - "24": 1801.0, - "25": 1665.0, - "26": 1869.0, - "27": 1901.0, - "28": 1874.0, - "29": 1934.0, - "30": 1835.0, - "31": 2054.0, - "32": 1967.0, - "33": 1859.0, - "34": 1998.0, - "35": 2070.0, - "36": 2010.0, - "37": 2252.0, - "38": 2135.0, - "39": 2189.0, - "40": 2277.0, - "41": 2394.0, - "42": 2163.0, - "43": 2421.0, - "44": 2333.0, - "45": 2614.0, - "46": 2537.0, - "47": 2494.0, - "48": 2705.0, - "49": 2869.0, - "50": 2689.0 + "1": 1806.0, + "2": 1666.0, + "3": 1783.0, + "4": 1693.0, + "5": 1654.0, + "6": 1608.0, + "7": 1873.0, + "8": 1698.0, + "9": 1693.0, + "10": 1821.0, + "11": 1696.0, + "12": 1683.0, + "13": 1736.0, + "14": 1800.0, + "15": 1622.0, + "16": 1836.0, + "17": 1808.0, + "18": 1795.0, + "19": 1633.0, + "20": 1618.0, + "21": 1776.0, + "22": 1658.0, + "23": 1687.0, + "24": 1786.0, + "25": 1651.0, + "26": 1907.0, + "27": 1859.0, + "28": 1873.0, + "29": 2007.0, + "30": 1769.0, + "31": 2107.0, + "32": 1936.0, + "33": 1999.0, + "34": 2014.0, + "35": 2117.0, + "36": 2011.0, + "37": 2288.0, + "38": 2159.0, + "39": 2281.0, + "40": 2350.0, + "41": 2358.0, + "42": 2100.0, + "43": 2369.0, + "44": 2298.0, + "45": 2651.0, + "46": 2400.0, + "47": 2499.0, + "48": 2720.0, + "49": 2940.0, + "50": 2656.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.79185, - "3": 0.1684, - "4": 0.30007, - "5": 0.41737, - "6": 0.61385, - "7": 0.4785, - "8": 0.56856, - "9": 0.89221, - "10": 0.31726, - "11": 0.62081, - "12": 0.44217, - "13": 0.3146, - "14": 0.49953, - "15": 0.31351, - "16": 0.5787, - "17": 0.39675, - "18": 0.49382, - "19": 0.25357, - "20": 0.67138, - "21": 0.34825, - "22": 0.39821, - "23": 0.33723, - "24": 0.36661, - "25": 0.65768, - "26": 0.56364, - "27": 0.49364, - "28": 0.32846, - "29": 0.3984, - "30": 0.37947, - "31": 0.19854, - "32": 0.38359, - "33": 0.40613, - "34": 0.44257, - "35": 0.69906, - "36": 0.3527, - "37": 0.36409, - "38": 0.47772, - "39": 0.38507, - "40": 0.34556, - "41": 0.44601, - "42": 0.38203, - "43": 0.4813, - "44": 0.31021, - "45": 0.3063, - "46": 0.3706, - "47": 0.40902, - "48": 0.40634, - "49": 0.63022, - "50": 0.52699 + "2": 2.8185, + "3": 0.13597, + "4": 0.48902, + "5": 0.48529, + "6": 1.13119, + "7": 0.64214, + "8": 0.5434, + "9": 0.47414, + "10": 0.44386, + "11": 0.35033, + "12": 0.44076, + "13": 0.35009, + "14": 0.74724, + "15": 0.37603, + "16": 0.58008, + "17": 0.66482, + "18": 0.29888, + "19": 0.28084, + "20": 0.37191, + "21": 0.66834, + "22": 0.35727, + "23": 0.68992, + "24": 0.28506, + "25": 0.4378, + "26": 0.73539, + "27": 0.66812, + "28": 0.42313, + "29": 0.40286, + "30": 0.36446, + "31": 0.17387, + "32": 0.69209, + "33": 0.17062, + "34": 0.2642, + "35": 0.68255, + "36": 0.50586, + "37": 0.41718, + "38": 0.32782, + "39": 0.62276, + "40": 0.29565, + "41": 0.42605, + "42": 0.42425, + "43": 0.35723, + "44": 0.6737, + "45": 0.25832, + "46": 0.62298, + "47": 0.47844, + "48": 0.42779, + "49": 0.55055, + "50": 0.38279 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 15d7bdfab04..95e9ea34685 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.8851, - "2": 10.88271, - "3": 10.88387, - "4": 10.88623, - "5": 10.87916, - "6": 10.87659, - "7": 10.87962, - "8": 10.87212, - "9": 10.88437, - "10": 10.87861, + "1": 10.88508, + "2": 10.88267, + "3": 10.88381, + "4": 10.88628, + "5": 10.87917, + "6": 10.87655, + "7": 10.87963, + "8": 10.87208, + "9": 10.88438, + "10": 10.87859, "11": 10.87351, - "12": 10.86322, - "13": 10.87875, - "14": 10.86597, - "15": 10.83786, - "16": 10.83261, - "17": 10.84042, - "18": 10.82308, - "19": 10.8357, - "20": 10.72914, - "21": 10.73909, - "22": 10.72511, - "23": 10.71158, - "24": 10.68285, - "25": 10.67542, - "26": 10.67984, - "27": 10.63495, - "28": 10.57987, + "12": 10.86335, + "13": 10.87873, + "14": 10.86596, + "15": 10.83787, + "16": 10.83256, + "17": 10.84044, + "18": 10.82313, + "19": 10.83568, + "20": 10.72916, + "21": 10.73922, + "22": 10.72505, + "23": 10.71164, + "24": 10.6829, + "25": 10.67538, + "26": 10.67983, + "27": 10.63488, + "28": 10.57997, "29": 10.54429, - "30": 10.52094, - "31": 10.51407, - "32": 10.49438, - "33": 10.46828, - "34": 10.43398, - "35": 10.43852, - "36": 10.4131, - "37": 10.3791, - "38": 10.39374, - "39": 10.35128, - "40": 10.34595, - "41": 10.32312, - "42": 10.29759, - "43": 10.28543, - "44": 10.24463, - "45": 10.27145, - "46": 10.22653, - "47": 10.22023, - "48": 10.17334, - "49": 10.17076, - "50": 10.17217, + "30": 10.52101, + "31": 10.51408, + "32": 10.49434, + "33": 10.46831, + "34": 10.43403, + "35": 10.4385, + "36": 10.41313, + "37": 10.37914, + "38": 10.39376, + "39": 10.35138, + "40": 10.34602, + "41": 10.32317, + "42": 10.29756, + "43": 10.28548, + "44": 10.24467, + "45": 10.27152, + "46": 10.22656, + "47": 10.2203, + "48": 10.17338, + "49": 10.1708, + "50": 10.17222, "51": 10.1777, - "52": 10.13082, - "53": 10.1398, - "54": 10.10412, - "55": 10.0701, - "56": 10.10629, - "57": 10.09822, - "58": 10.1116, - "59": 10.0548, - "60": 10.07318, - "61": 10.02711, - "62": 9.99676, - "63": 10.07084, - "64": 10.02892, - "65": 10.00122, - "66": 10.02707, - "67": 10.00255, - "68": 9.96484, + "52": 10.13089, + "53": 10.13982, + "54": 10.10413, + "55": 10.07015, + "56": 10.10634, + "57": 10.0982, + "58": 10.11163, + "59": 10.05487, + "60": 10.07321, + "61": 10.02714, + "62": 9.99681, + "63": 10.07086, + "64": 10.02893, + "65": 10.00125, + "66": 10.02713, + "67": 10.00263, + "68": 9.96482, "69": 9.98873, - "70": 9.97572, - "71": 9.9981, - "72": 9.97566, - "73": 9.96388, - "74": 9.95687, - "75": 9.92479, - "76": 9.9637, - "77": 9.95559, - "78": 9.90277, - "79": 9.90952, - "80": 9.92737, - "81": 9.94386, - "82": 9.89003, - "83": 9.84869, - "84": 9.78551, - "85": 9.77693, - "86": 9.87452, - "87": 9.90795, - "88": 9.88609, + "70": 9.97571, + "71": 9.99817, + "72": 9.97571, + "73": 9.96387, + "74": 9.9569, + "75": 9.92482, + "76": 9.96373, + "77": 9.95564, + "78": 9.90284, + "79": 9.90955, + "80": 9.92744, + "81": 9.94394, + "82": 9.89006, + "83": 9.84876, + "84": 9.78554, + "85": 9.77698, + "86": 9.87455, + "87": 9.908, + "88": 9.88614, "89": 9.81901, - "90": 9.81049, - "91": 9.82156, - "92": 9.81513, - "93": 9.7511, - "94": 9.82585, - "95": 9.82052, - "96": 9.80342, - "97": 9.74064, - "98": 9.77418, - "99": 9.81987, - "100": 9.71431 + "90": 9.81052, + "91": 9.82161, + "92": 9.81514, + "93": 9.75109, + "94": 9.8259, + "95": 9.82056, + "96": 9.80345, + "97": 9.74073, + "98": 9.77421, + "99": 9.8199, + "100": 9.71432 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1709.0, - "2": 1738.0, - "3": 1755.0, - "4": 1816.0, - "5": 1708.0, - "6": 1634.0, - "7": 1838.0, - "8": 1695.0, - "9": 1767.0, - "10": 1770.0, - "11": 1646.0, - "12": 1694.0, - "13": 1827.0, - "14": 1836.0, - "15": 1668.0, - "16": 1771.0, - "17": 1710.0, - "18": 1810.0, - "19": 1626.0, - "20": 1637.0, - "21": 1755.0, - "22": 1610.0, - "23": 1699.0, - "24": 1736.0, - "25": 1665.0, - "26": 1825.0, - "27": 1873.0, - "28": 1873.0, - "29": 1950.0, - "30": 1801.0, - "31": 2007.0, - "32": 1957.0, - "33": 1989.0, - "34": 2056.0, - "35": 1989.0, - "36": 1986.0, - "37": 2275.0, - "38": 2141.0, - "39": 2165.0, - "40": 2317.0, - "41": 2404.0, - "42": 2080.0, - "43": 2426.0, - "44": 2321.0, - "45": 2630.0, - "46": 2483.0, - "47": 2580.0, - "48": 2699.0, - "49": 2889.0, - "50": 2667.0, - "51": 2617.0, - "52": 2850.0, - "53": 2625.0, - "54": 3028.0, - "55": 2634.0, - "56": 2766.0, - "57": 2237.0, - "58": 3612.0, - "59": 2973.0, - "60": 2999.0, - "61": 2852.0, - "62": 3267.0, - "63": 3413.0, - "64": 3691.0, - "65": 2757.0, - "66": 3276.0, - "67": 3939.0, - "68": 3751.0, - "69": 3041.0, - "70": 3416.0, - "71": 3101.0, - "72": 3182.0, - "73": 3409.0, - "74": 3380.0, - "75": 3196.0, - "76": 3354.0, - "77": 3776.0, - "78": 3164.0, - "79": 3305.0, - "80": 2976.0, - "81": 3507.0, - "82": 3083.0, - "83": 3097.0, - "84": 3179.0, - "85": 2726.0, - "86": 3155.0, - "87": 2915.0, - "88": 2976.0, - "89": 2918.0, - "90": 3555.0, - "91": 3025.0, - "92": 3105.0, - "93": 3130.0, - "94": 2975.0, - "95": 3522.0, - "96": 3262.0, - "97": 3671.0, - "98": 3711.0, - "99": 3330.0, - "100": 3251.0 + "1": 1824.0, + "2": 1713.0, + "3": 1872.0, + "4": 1690.0, + "5": 1686.0, + "6": 1613.0, + "7": 1828.0, + "8": 1722.0, + "9": 1716.0, + "10": 1818.0, + "11": 1631.0, + "12": 1669.0, + "13": 1800.0, + "14": 1755.0, + "15": 1631.0, + "16": 1721.0, + "17": 1856.0, + "18": 1900.0, + "19": 1636.0, + "20": 1772.0, + "21": 1802.0, + "22": 1641.0, + "23": 1666.0, + "24": 1872.0, + "25": 1760.0, + "26": 1829.0, + "27": 1889.0, + "28": 1815.0, + "29": 1898.0, + "30": 1847.0, + "31": 1969.0, + "32": 1980.0, + "33": 1915.0, + "34": 2069.0, + "35": 1997.0, + "36": 2015.0, + "37": 2151.0, + "38": 2117.0, + "39": 2171.0, + "40": 2211.0, + "41": 2322.0, + "42": 2104.0, + "43": 2449.0, + "44": 2326.0, + "45": 2713.0, + "46": 2396.0, + "47": 2602.0, + "48": 2663.0, + "49": 2932.0, + "50": 2675.0, + "51": 2588.0, + "52": 2858.0, + "53": 2618.0, + "54": 3000.0, + "55": 2710.0, + "56": 2785.0, + "57": 2223.0, + "58": 3664.0, + "59": 2943.0, + "60": 2942.0, + "61": 2776.0, + "62": 3178.0, + "63": 3484.0, + "64": 3783.0, + "65": 2726.0, + "66": 3229.0, + "67": 3944.0, + "68": 3708.0, + "69": 3014.0, + "70": 3465.0, + "71": 3161.0, + "72": 3092.0, + "73": 3469.0, + "74": 3377.0, + "75": 3166.0, + "76": 3304.0, + "77": 3749.0, + "78": 3340.0, + "79": 3161.0, + "80": 3000.0, + "81": 3578.0, + "82": 3152.0, + "83": 3167.0, + "84": 3054.0, + "85": 2736.0, + "86": 3135.0, + "87": 2983.0, + "88": 3049.0, + "89": 2953.0, + "90": 3523.0, + "91": 2958.0, + "92": 3151.0, + "93": 3191.0, + "94": 3151.0, + "95": 3498.0, + "96": 3354.0, + "97": 3654.0, + "98": 3692.0, + "99": 3368.0, + "100": 3213.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.76083, - "3": 0.1392, - "4": 0.72471, - "5": 0.37212, - "6": 0.65496, - "7": 0.81145, - "8": 1.18276, - "9": 0.74068, - "10": 0.62362, - "11": 0.90016, - "12": 0.3931, - "13": 0.66456, - "14": 0.54956, - "15": 0.5344, - "16": 0.56924, - "17": 0.48379, - "18": 0.82846, - "19": 0.52198, - "20": 0.30559, - "21": 0.68469, - "22": 0.6055, - "23": 0.6241, - "24": 0.74674, - "25": 0.14371, - "26": 0.74475, - "27": 0.24564, - "28": 0.64924, - "29": 0.62848, - "30": 0.15121, - "31": 0.51775, - "32": 1.05545, - "33": 0.36409, - "34": 0.82885, - "35": 0.18612, - "36": 0.70268, - "37": 0.53875, - "38": 0.41068, - "39": 0.37853, - "40": 0.56174, - "41": 0.71995, - "42": 0.46478, - "43": 0.48267, - "44": 1.00388, - "45": 0.19292, - "46": 0.6001, - "47": 0.51524, - "48": 0.72325, - "49": 0.4162, - "50": 0.39522, - "51": 0.17241, - "52": 0.61332, - "53": 0.52274, - "54": 0.74178, - "55": 0.54106, - "56": 0.80779, - "57": 0.43858, - "58": 0.51628, - "59": 0.72496, - "60": 0.66828, - "61": 0.27282, - "62": 0.75232, - "63": 0.56444, - "64": 0.65677, - "65": 0.48561, - "66": 0.80335, - "67": 0.65147, - "68": 0.15498, - "69": 0.42658, - "70": 1.05728, - "71": 0.2845, - "72": 0.64147, - "73": 0.57041, - "74": 0.44787, - "75": 0.97937, - "76": 0.45986, - "77": 0.41157, - "78": 1.43496, - "79": 0.3979, - "80": 0.55705, - "81": 0.65481, - "82": 0.3819, - "83": 0.66336, - "84": 0.74797, - "85": 0.14404, - "86": 0.46607, - "87": 1.04081, - "88": 0.53513, - "89": 0.79143, - "90": 0.46305, - "91": 0.49708, - "92": 0.70755, - "93": 0.25051, - "94": 0.58557, - "95": 1.0549, - "96": 0.4588, - "97": 0.58513, - "98": 0.64325, - "99": 1.33531, - "100": 0.30734 + "2": 4.13223, + "3": 0.12081, + "4": 0.70244, + "5": 0.37983, + "6": 0.61351, + "7": 0.83049, + "8": 0.91549, + "9": 0.59545, + "10": 0.36477, + "11": 0.88171, + "12": 0.36995, + "13": 0.59663, + "14": 0.52992, + "15": 0.51036, + "16": 0.54535, + "17": 0.52034, + "18": 0.66754, + "19": 0.51503, + "20": 0.3268, + "21": 0.68257, + "22": 0.55035, + "23": 0.59017, + "24": 0.77686, + "25": 0.12597, + "26": 0.83118, + "27": 0.25333, + "28": 0.52887, + "29": 0.63362, + "30": 0.1345, + "31": 0.38365, + "32": 0.94833, + "33": 0.34237, + "34": 0.73988, + "35": 0.2318, + "36": 0.63713, + "37": 0.54599, + "38": 0.38193, + "39": 0.56756, + "40": 0.71986, + "41": 0.48118, + "42": 0.4737, + "43": 0.49991, + "44": 0.58917, + "45": 0.57502, + "46": 0.39614, + "47": 0.51242, + "48": 0.67699, + "49": 0.38866, + "50": 0.45961, + "51": 0.15647, + "52": 0.50737, + "53": 0.52885, + "54": 0.76693, + "55": 0.71316, + "56": 0.81237, + "57": 0.41483, + "58": 0.88969, + "59": 0.52862, + "60": 0.63381, + "61": 0.38425, + "62": 1.03211, + "63": 0.20876, + "64": 0.9013, + "65": 0.42769, + "66": 0.58466, + "67": 0.60168, + "68": 0.16026, + "69": 0.7312, + "70": 0.91781, + "71": 0.26044, + "72": 0.63034, + "73": 0.55519, + "74": 0.40821, + "75": 0.95778, + "76": 0.43492, + "77": 0.40932, + "78": 1.29287, + "79": 0.32529, + "80": 0.54963, + "81": 0.59584, + "82": 0.72511, + "83": 0.24128, + "84": 0.71065, + "85": 0.3183, + "86": 0.46947, + "87": 1.05355, + "88": 0.71611, + "89": 0.67109, + "90": 0.37655, + "91": 0.48364, + "92": 0.68176, + "93": 0.24045, + "94": 0.66091, + "95": 0.86017, + "96": 0.34943, + "97": 0.58827, + "98": 0.57889, + "99": 1.2659, + "100": 0.29593 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json index 00ff516f228..6b4748fed26 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.88423, - "2": 10.88416, - "3": 10.88528, - "4": 10.88261, - "5": 10.87548, - "6": 10.87734, - "7": 10.87899, - "8": 10.87648, - "9": 10.88484, - "10": 10.87621, - "11": 10.87474, - "12": 10.86383, - "13": 10.87811, - "14": 10.86679, - "15": 10.83811, - "16": 10.82851, - "17": 10.84398, - "18": 10.8262, - "19": 10.83051, - "20": 10.73063, - "21": 10.7376, - "22": 10.72457, - "23": 10.71467, - "24": 10.68914, - "25": 10.67536, - "26": 10.6774, - "27": 10.63672, - "28": 10.58124, - "29": 10.54372, - "30": 10.52131, - "31": 10.51635, - "32": 10.49663, - "33": 10.46952, - "34": 10.43406, - "35": 10.44014, - "36": 10.4129, - "37": 10.37742, - "38": 10.39312, - "39": 10.35169, - "40": 10.34761, - "41": 10.32369, + "1": 10.88426, + "2": 10.88414, + "3": 10.88523, + "4": 10.88255, + "5": 10.87555, + "6": 10.87737, + "7": 10.87889, + "8": 10.87647, + "9": 10.88483, + "10": 10.87617, + "11": 10.8747, + "12": 10.86391, + "13": 10.87812, + "14": 10.8668, + "15": 10.83802, + "16": 10.82846, + "17": 10.84396, + "18": 10.82611, + "19": 10.83045, + "20": 10.7307, + "21": 10.73767, + "22": 10.72458, + "23": 10.71466, + "24": 10.68906, + "25": 10.67538, + "26": 10.6775, + "27": 10.63684, + "28": 10.58134, + "29": 10.54385, + "30": 10.52138, + "31": 10.51634, + "32": 10.49671, + "33": 10.46958, + "34": 10.43408, + "35": 10.44022, + "36": 10.41294, + "37": 10.37743, + "38": 10.39318, + "39": 10.3517, + "40": 10.34764, + "41": 10.32375, "42": 10.29737, - "43": 10.28796, - "44": 10.24485, - "45": 10.27231, - "46": 10.22856, - "47": 10.22097, - "48": 10.17292, - "49": 10.17266, - "50": 10.17276, - "51": 10.177, - "52": 10.12982, - "53": 10.14031, - "54": 10.10481, - "55": 10.07099, - "56": 10.10607, - "57": 10.09836, - "58": 10.11273, - "59": 10.05586, - "60": 10.07207, - "61": 10.0265, - "62": 9.99707, - "63": 10.07081, - "64": 10.02756, - "65": 10.0026, + "43": 10.28799, + "44": 10.24488, + "45": 10.27232, + "46": 10.22865, + "47": 10.22099, + "48": 10.17298, + "49": 10.17272, + "50": 10.17282, + "51": 10.17703, + "52": 10.1299, + "53": 10.1403, + "54": 10.10484, + "55": 10.071, + "56": 10.10606, + "57": 10.09847, + "58": 10.11276, + "59": 10.05585, + "60": 10.07216, + "61": 10.02653, + "62": 9.99717, + "63": 10.07085, + "64": 10.02754, + "65": 10.00264, "66": 10.02863, - "67": 10.00301, - "68": 9.96495, - "69": 9.98889, - "70": 9.97585, - "71": 9.99932, - "72": 9.97696, - "73": 9.96491, - "74": 9.95749, - "75": 9.92624, - "76": 9.96521, - "77": 9.95724, - "78": 9.90485, - "79": 9.91038, - "80": 9.9292, - "81": 9.94483, - "82": 9.88812, - "83": 9.84979, + "67": 10.00302, + "68": 9.96496, + "69": 9.98896, + "70": 9.97581, + "71": 9.9994, + "72": 9.97697, + "73": 9.96497, + "74": 9.95752, + "75": 9.92629, + "76": 9.96522, + "77": 9.95726, + "78": 9.90488, + "79": 9.91046, + "80": 9.92924, + "81": 9.94488, + "82": 9.88813, + "83": 9.84984, "84": 9.78547, - "85": 9.77729, - "86": 9.87458, - "87": 9.9087, - "88": 9.88616, - "89": 9.81986, - "90": 9.81011, - "91": 9.8223, - "92": 9.81413, - "93": 9.75326, - "94": 9.82541, - "95": 9.8214, - "96": 9.80427, - "97": 9.74114, - "98": 9.77482, - "99": 9.81968, - "100": 9.71395 + "85": 9.77734, + "86": 9.87459, + "87": 9.90872, + "88": 9.88617, + "89": 9.81991, + "90": 9.81017, + "91": 9.82232, + "92": 9.81417, + "93": 9.75331, + "94": 9.82547, + "95": 9.82143, + "96": 9.80432, + "97": 9.74116, + "98": 9.77486, + "99": 9.81971, + "100": 9.71399 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1843.0, - "2": 1668.0, - "3": 1804.0, - "4": 1839.0, - "5": 1800.0, - "6": 1703.0, - "7": 2038.0, - "8": 1748.0, - "9": 1864.0, - "10": 1842.0, - "11": 1719.0, - "12": 1714.0, - "13": 1813.0, - "14": 1909.0, - "15": 1666.0, - "16": 1677.0, - "17": 1796.0, - "18": 1836.0, - "19": 1703.0, - "20": 1726.0, - "21": 1839.0, - "22": 1813.0, - "23": 1788.0, - "24": 1840.0, - "25": 1787.0, - "26": 1950.0, - "27": 1875.0, - "28": 1870.0, - "29": 2011.0, - "30": 1861.0, - "31": 2072.0, - "32": 1968.0, - "33": 2036.0, - "34": 2073.0, - "35": 2124.0, - "36": 2149.0, - "37": 2296.0, - "38": 2106.0, - "39": 2219.0, - "40": 2429.0, - "41": 2501.0, - "42": 2082.0, - "43": 2440.0, - "44": 2346.0, - "45": 2665.0, - "46": 2495.0, - "47": 2635.0, - "48": 2718.0, - "49": 3000.0, - "50": 2690.0, - "51": 2658.0, - "52": 2930.0, - "53": 2647.0, - "54": 3119.0, - "55": 2760.0, - "56": 2818.0, - "57": 2277.0, - "58": 3718.0, - "59": 2994.0, - "60": 3005.0, - "61": 2932.0, - "62": 3445.0, - "63": 3408.0, - "64": 3883.0, - "65": 2810.0, - "66": 3252.0, - "67": 4018.0, - "68": 3613.0, - "69": 3005.0, - "70": 3461.0, - "71": 3272.0, - "72": 3026.0, - "73": 3627.0, - "74": 3493.0, - "75": 3285.0, - "76": 3303.0, - "77": 3760.0, - "78": 3414.0, - "79": 3400.0, - "80": 3078.0, - "81": 3574.0, - "82": 2969.0, - "83": 3222.0, - "84": 3028.0, - "85": 2914.0, - "86": 3177.0, - "87": 3086.0, - "88": 3122.0, - "89": 3130.0, - "90": 3530.0, - "91": 2871.0, - "92": 3204.0, - "93": 3193.0, - "94": 3259.0, - "95": 3558.0, - "96": 3454.0, - "97": 3618.0, - "98": 3588.0, - "99": 3350.0, - "100": 3265.0 + "1": 1759.0, + "2": 1797.0, + "3": 1766.0, + "4": 1748.0, + "5": 1792.0, + "6": 1712.0, + "7": 1920.0, + "8": 1661.0, + "9": 1769.0, + "10": 1767.0, + "11": 1759.0, + "12": 1734.0, + "13": 1781.0, + "14": 1856.0, + "15": 1599.0, + "16": 1767.0, + "17": 1781.0, + "18": 1814.0, + "19": 1683.0, + "20": 1813.0, + "21": 1883.0, + "22": 1678.0, + "23": 1715.0, + "24": 1775.0, + "25": 1732.0, + "26": 1937.0, + "27": 1899.0, + "28": 1866.0, + "29": 1937.0, + "30": 1855.0, + "31": 2038.0, + "32": 1973.0, + "33": 2013.0, + "34": 2133.0, + "35": 2135.0, + "36": 2118.0, + "37": 2348.0, + "38": 2205.0, + "39": 2211.0, + "40": 2389.0, + "41": 2352.0, + "42": 2139.0, + "43": 2537.0, + "44": 2303.0, + "45": 2657.0, + "46": 2535.0, + "47": 2524.0, + "48": 2710.0, + "49": 3010.0, + "50": 2771.0, + "51": 2610.0, + "52": 2770.0, + "53": 2670.0, + "54": 2949.0, + "55": 2729.0, + "56": 2885.0, + "57": 2347.0, + "58": 3626.0, + "59": 2956.0, + "60": 2918.0, + "61": 2943.0, + "62": 3475.0, + "63": 3477.0, + "64": 3819.0, + "65": 2843.0, + "66": 3207.0, + "67": 3868.0, + "68": 3649.0, + "69": 2959.0, + "70": 3542.0, + "71": 3087.0, + "72": 3087.0, + "73": 3558.0, + "74": 3507.0, + "75": 3209.0, + "76": 3278.0, + "77": 3784.0, + "78": 3311.0, + "79": 3369.0, + "80": 3055.0, + "81": 3598.0, + "82": 3002.0, + "83": 3220.0, + "84": 3079.0, + "85": 3002.0, + "86": 3128.0, + "87": 3158.0, + "88": 3044.0, + "89": 3139.0, + "90": 3465.0, + "91": 2839.0, + "92": 3225.0, + "93": 3339.0, + "94": 3272.0, + "95": 3593.0, + "96": 3279.0, + "97": 3730.0, + "98": 3631.0, + "99": 3314.0, + "100": 3409.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1260800512.0, - "2": 1260800512.0, - "3": 1260800512.0, - "4": 1260800512.0, - "5": 1260800512.0, - "6": 1260800512.0, - "7": 1260800512.0, - "8": 1260800512.0, - "9": 1260800512.0, - "10": 1260800512.0, - "11": 1260800512.0, - "12": 1260800512.0, - "13": 1260800512.0, - "14": 1260800512.0, - "15": 1260800512.0, - "16": 1260800512.0, - "17": 1260800512.0, - "18": 1260800512.0, - "19": 1260800512.0, - "20": 1260800512.0, - "21": 1260800512.0, - "22": 1260800512.0, - "23": 1260800512.0, - "24": 1260800512.0, - "25": 1260800512.0, - "26": 1260800512.0, - "27": 1260800512.0, - "28": 1260800512.0, - "29": 1260800512.0, - "30": 1260800512.0, - "31": 1260800512.0, - "32": 1260800512.0, - "33": 1260800512.0, - "34": 1260800512.0, - "35": 1260800512.0, - "36": 1260800512.0, - "37": 1260800512.0, - "38": 1260800512.0, - "39": 1260800512.0, - "40": 1260800512.0, - "41": 1260800512.0, - "42": 1260800512.0, - "43": 1260800512.0, - "44": 1260800512.0, - "45": 1260800512.0, - "46": 1260800512.0, - "47": 1260800512.0, - "48": 1260800512.0, - "49": 1260800512.0, - "50": 1260800512.0, - "51": 1260800512.0, - "52": 1260800512.0, - "53": 1260800512.0, - "54": 1260800512.0, - "55": 1260800512.0, - "56": 1260800512.0, - "57": 1260800512.0, - "58": 1260800512.0, - "59": 1260800512.0, - "60": 1260800512.0, - "61": 1260800512.0, - "62": 1260800512.0, - "63": 1260800512.0, - "64": 1260800512.0, - "65": 1260800512.0, - "66": 1260800512.0, - "67": 1260800512.0, - "68": 1260800512.0, - "69": 1260800512.0, - "70": 1260800512.0, - "71": 1260800512.0, - "72": 1260800512.0, - "73": 1260800512.0, - "74": 1260800512.0, - "75": 1260800512.0, - "76": 1260800512.0, - "77": 1260800512.0, - "78": 1260800512.0, - "79": 1260800512.0, - "80": 1260800512.0, - "81": 1260800512.0, - "82": 1260800512.0, - "83": 1260800512.0, - "84": 1260800512.0, - "85": 1260800512.0, - "86": 1260800512.0, - "87": 1260800512.0, - "88": 1260800512.0, - "89": 1260800512.0, - "90": 1260800512.0, - "91": 1260800512.0, - "92": 1260800512.0, - "93": 1260800512.0, - "94": 1260800512.0, - "95": 1260800512.0, - "96": 1260800512.0, - "97": 1260800512.0, - "98": 1260800512.0, - "99": 1260800512.0, - "100": 1260800512.0 + "1": 1259751936.0, + "2": 1259751936.0, + "3": 1259751936.0, + "4": 1259751936.0, + "5": 1259751936.0, + "6": 1259751936.0, + "7": 1259751936.0, + "8": 1259751936.0, + "9": 1259751936.0, + "10": 1259751936.0, + "11": 1259751936.0, + "12": 1259751936.0, + "13": 1259751936.0, + "14": 1259751936.0, + "15": 1259751936.0, + "16": 1259751936.0, + "17": 1259751936.0, + "18": 1259751936.0, + "19": 1259751936.0, + "20": 1259751936.0, + "21": 1259751936.0, + "22": 1259751936.0, + "23": 1259751936.0, + "24": 1259751936.0, + "25": 1259751936.0, + "26": 1259751936.0, + "27": 1259751936.0, + "28": 1259751936.0, + "29": 1259751936.0, + "30": 1259751936.0, + "31": 1259751936.0, + "32": 1259751936.0, + "33": 1259751936.0, + "34": 1259751936.0, + "35": 1259751936.0, + "36": 1259751936.0, + "37": 1259751936.0, + "38": 1259751936.0, + "39": 1259751936.0, + "40": 1259751936.0, + "41": 1259751936.0, + "42": 1259751936.0, + "43": 1259751936.0, + "44": 1259751936.0, + "45": 1259751936.0, + "46": 1259751936.0, + "47": 1259751936.0, + "48": 1259751936.0, + "49": 1259751936.0, + "50": 1259751936.0, + "51": 1259751936.0, + "52": 1259751936.0, + "53": 1259751936.0, + "54": 1259751936.0, + "55": 1259751936.0, + "56": 1259751936.0, + "57": 1259751936.0, + "58": 1259751936.0, + "59": 1259751936.0, + "60": 1259751936.0, + "61": 1259751936.0, + "62": 1259751936.0, + "63": 1259751936.0, + "64": 1259751936.0, + "65": 1259751936.0, + "66": 1259751936.0, + "67": 1259751936.0, + "68": 1259751936.0, + "69": 1259751936.0, + "70": 1259751936.0, + "71": 1259751936.0, + "72": 1259751936.0, + "73": 1259751936.0, + "74": 1259751936.0, + "75": 1259751936.0, + "76": 1259751936.0, + "77": 1259751936.0, + "78": 1259751936.0, + "79": 1259751936.0, + "80": 1259751936.0, + "81": 1259751936.0, + "82": 1259751936.0, + "83": 1259751936.0, + "84": 1259751936.0, + "85": 1259751936.0, + "86": 1259751936.0, + "87": 1259751936.0, + "88": 1259751936.0, + "89": 1259751936.0, + "90": 1259751936.0, + "91": 1259751936.0, + "92": 1259751936.0, + "93": 1259751936.0, + "94": 1259751936.0, + "95": 1259751936.0, + "96": 1259751936.0, + "97": 1259751936.0, + "98": 1259751936.0, + "99": 1259751936.0, + "100": 1259751936.0 } }, "mem-max-allocated-bytes": { @@ -326,55 +326,55 @@ "step_interval": 1, "values": { "1": 2013853696.0, - "2": 2562382848.0, - "3": 2562382848.0, - "4": 2562382848.0, - "5": 2562382848.0, - "6": 2562382848.0, - "7": 2562382848.0, - "8": 2562382848.0, - "9": 2562382848.0, - "10": 2562382848.0, - "11": 2562382848.0, - "12": 2562382848.0, - "13": 2562382848.0, - "14": 2562382848.0, - "15": 2562382848.0, - "16": 2562382848.0, - "17": 2562382848.0, - "18": 2562382848.0, - "19": 2562382848.0, - "20": 2562382848.0, - "21": 2562382848.0, - "22": 2562382848.0, - "23": 2562382848.0, - "24": 2562382848.0, - "25": 2562382848.0, - "26": 2562382848.0, - "27": 2562382848.0, - "28": 2562382848.0, - "29": 2562382848.0, - "30": 2562382848.0, - "31": 2562382848.0, - "32": 2562382848.0, - "33": 2562382848.0, - "34": 2562382848.0, - "35": 2562382848.0, - "36": 2562382848.0, - "37": 2562382848.0, - "38": 2562382848.0, - "39": 2562382848.0, - "40": 2562382848.0, - "41": 2562382848.0, - "42": 2562382848.0, - "43": 2562382848.0, - "44": 2562382848.0, - "45": 2562382848.0, - "46": 2562382848.0, - "47": 2562382848.0, - "48": 2562382848.0, - "49": 2562382848.0, - "50": 2562382848.0, + "2": 2561334272.0, + "3": 2561334272.0, + "4": 2561334272.0, + "5": 2561334272.0, + "6": 2561334272.0, + "7": 2561334272.0, + "8": 2561334272.0, + "9": 2561334272.0, + "10": 2561334272.0, + "11": 2561334272.0, + "12": 2561334272.0, + "13": 2561334272.0, + "14": 2561334272.0, + "15": 2561334272.0, + "16": 2561334272.0, + "17": 2561334272.0, + "18": 2561334272.0, + "19": 2561334272.0, + "20": 2561334272.0, + "21": 2561334272.0, + "22": 2561334272.0, + "23": 2561334272.0, + "24": 2561334272.0, + "25": 2561334272.0, + "26": 2561334272.0, + "27": 2561334272.0, + "28": 2561334272.0, + "29": 2561334272.0, + "30": 2561334272.0, + "31": 2561334272.0, + "32": 2561334272.0, + "33": 2561334272.0, + "34": 2561334272.0, + "35": 2561334272.0, + "36": 2561334272.0, + "37": 2561334272.0, + "38": 2561334272.0, + "39": 2561334272.0, + "40": 2561334272.0, + "41": 2561334272.0, + "42": 2561334272.0, + "43": 2561334272.0, + "44": 2561334272.0, + "45": 2561334272.0, + "46": 2561334272.0, + "47": 2561334272.0, + "48": 2561334272.0, + "49": 2561334272.0, + "50": 2561334272.0, "51": 2562382848.0, "52": 2562382848.0, "53": 2562382848.0, @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.14377, - "3": 0.13165, - "4": 0.11497, - "5": 0.11487, - "6": 0.11484, - "7": 0.1145, - "8": 0.11492, - "9": 0.11592, - "10": 0.1146, - "11": 0.11474, - "12": 0.11424, - "13": 0.11519, - "14": 0.11539, - "15": 0.11501, - "16": 0.12021, - "17": 0.11525, - "18": 0.116, - "19": 0.11472, - "20": 0.11462, - "21": 0.11496, - "22": 0.11611, - "23": 0.11736, - "24": 0.11653, - "25": 0.11503, - "26": 0.11581, - "27": 0.11579, - "28": 0.11552, - "29": 0.1166, - "30": 0.11553, - "31": 0.11484, - "32": 0.11471, - "33": 0.11515, - "34": 0.11548, - "35": 0.11495, - "36": 0.11532, - "37": 0.11624, - "38": 0.11636, - "39": 0.11576, - "40": 0.11607, - "41": 0.11491, - "42": 0.11477, - "43": 0.11621, - "44": 0.11623, - "45": 0.11667, - "46": 0.11682, - "47": 0.11644, - "48": 0.1162, - "49": 0.11674, - "50": 0.11816, - "51": 0.13233, - "52": 0.13722, - "53": 0.16225, - "54": 0.12906, - "55": 0.12035, - "56": 0.11722, - "57": 0.11624, - "58": 0.11822, - "59": 0.11699, - "60": 0.11624, - "61": 0.11595, - "62": 0.11737, - "63": 0.11601, - "64": 0.11699, - "65": 0.11675, - "66": 0.11674, - "67": 0.1165, - "68": 0.11686, - "69": 0.11548, - "70": 0.11755, - "71": 0.11594, - "72": 0.11681, - "73": 0.11646, - "74": 0.11656, - "75": 0.11679, - "76": 0.11734, - "77": 0.11805, - "78": 0.11788, - "79": 0.11877, - "80": 0.11746, - "81": 0.11677, - "82": 0.11685, - "83": 0.11801, - "84": 0.11945, - "85": 0.11699, - "86": 0.11797, - "87": 0.11741, - "88": 0.11698, - "89": 0.11728, - "90": 0.11831, - "91": 0.11826, - "92": 0.11736, - "93": 0.11748, - "94": 0.11804, - "95": 0.11813, - "96": 0.11768, - "97": 0.11728, - "98": 0.11759, - "99": 0.11767, - "100": 0.11758 + "2": 4.09418, + "3": 0.14875, + "4": 0.10138, + "5": 0.10407, + "6": 0.10329, + "7": 0.10353, + "8": 0.10474, + "9": 0.10512, + "10": 0.10479, + "11": 0.10493, + "12": 0.10467, + "13": 0.10382, + "14": 0.10416, + "15": 0.10464, + "16": 0.1048, + "17": 0.10737, + "18": 0.10759, + "19": 0.10705, + "20": 0.10612, + "21": 0.1064, + "22": 0.10526, + "23": 0.10501, + "24": 0.10629, + "25": 0.1067, + "26": 0.10576, + "27": 0.10786, + "28": 0.1078, + "29": 0.10887, + "30": 0.10965, + "31": 0.109, + "32": 0.10545, + "33": 0.10419, + "34": 0.14837, + "35": 0.10675, + "36": 0.11009, + "37": 0.11032, + "38": 0.11004, + "39": 0.10977, + "40": 0.11096, + "41": 0.11014, + "42": 0.11293, + "43": 0.11169, + "44": 0.11168, + "45": 0.10908, + "46": 0.10943, + "47": 0.11026, + "48": 0.11011, + "49": 0.11052, + "50": 0.10992, + "51": 0.14299, + "52": 0.12477, + "53": 0.14971, + "54": 0.10599, + "55": 0.10595, + "56": 0.1075, + "57": 0.10654, + "58": 0.10481, + "59": 0.10684, + "60": 0.10569, + "61": 0.10713, + "62": 0.10635, + "63": 0.10631, + "64": 0.10714, + "65": 0.106, + "66": 0.10534, + "67": 0.10581, + "68": 0.1064, + "69": 0.10747, + "70": 0.10681, + "71": 0.10625, + "72": 0.10533, + "73": 0.10491, + "74": 0.10575, + "75": 0.10429, + "76": 0.1062, + "77": 0.10926, + "78": 0.10881, + "79": 0.10776, + "80": 0.10771, + "81": 0.10736, + "82": 0.10733, + "83": 0.10771, + "84": 0.10593, + "85": 0.10533, + "86": 0.10475, + "87": 0.10498, + "88": 0.10612, + "89": 0.10651, + "90": 0.10599, + "91": 0.10548, + "92": 0.10538, + "93": 0.10544, + "94": 0.10518, + "95": 0.1036, + "96": 0.10571, + "97": 0.10813, + "98": 0.10709, + "99": 0.10844, + "100": 0.10796 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json index 0e05742094a..f2f1326e90f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93345, - "2": 10.92442, - "3": 10.92705, - "4": 10.93095, - "5": 10.93662, - "6": 10.92967, - "7": 10.92359, - "8": 10.92254, - "9": 10.92459, - "10": 10.92964, - "11": 10.90935, - "12": 10.9283, - "13": 10.89685, - "14": 10.89709, - "15": 10.89153, - "16": 10.87452, - "17": 10.87088, - "18": 10.87178, + "1": 10.93341, + "2": 10.92431, + "3": 10.92704, + "4": 10.93087, + "5": 10.93656, + "6": 10.92961, + "7": 10.9236, + "8": 10.92262, + "9": 10.92457, + "10": 10.92966, + "11": 10.90934, + "12": 10.92833, + "13": 10.89681, + "14": 10.89713, + "15": 10.89155, + "16": 10.87453, + "17": 10.8709, + "18": 10.87184, "19": 10.85369, - "20": 10.8046, - "21": 10.78432, - "22": 10.76481, - "23": 10.77092, - "24": 10.75018, - "25": 10.74709, + "20": 10.80471, + "21": 10.78433, + "22": 10.76479, + "23": 10.77096, + "24": 10.75026, + "25": 10.7471, "26": 10.72265, - "27": 10.68797, - "28": 10.61498, - "29": 10.59319, - "30": 10.55627, - "31": 10.56492, - "32": 10.54516, - "33": 10.51564, - "34": 10.48222, - "35": 10.48502, - "36": 10.46535, - "37": 10.42861, - "38": 10.42594, - "39": 10.39452, - "40": 10.3745, - "41": 10.3512, - "42": 10.34808, - "43": 10.31967, - "44": 10.29914, - "45": 10.30312, - "46": 10.26444, - "47": 10.25216, - "48": 10.21538, - "49": 10.202, - "50": 10.21083, - "51": 10.19946, - "52": 10.15802, - "53": 10.1637, - "54": 10.12833, - "55": 10.10244, - "56": 10.12752, - "57": 10.11583, - "58": 10.11961, - "59": 10.06695, - "60": 10.09292, - "61": 10.04269, - "62": 10.01398, - "63": 10.08126, - "64": 10.03224, - "65": 10.00032, - "66": 10.04316, - "67": 10.01552, - "68": 9.98467, - "69": 9.99675, + "27": 10.68792, + "28": 10.61507, + "29": 10.59321, + "30": 10.55636, + "31": 10.56495, + "32": 10.54518, + "33": 10.51569, + "34": 10.48225, + "35": 10.48504, + "36": 10.46545, + "37": 10.42864, + "38": 10.42608, + "39": 10.3946, + "40": 10.37454, + "41": 10.35122, + "42": 10.3481, + "43": 10.31972, + "44": 10.29918, + "45": 10.30317, + "46": 10.26454, + "47": 10.25224, + "48": 10.21545, + "49": 10.2021, + "50": 10.21085, + "51": 10.19943, + "52": 10.15812, + "53": 10.16371, + "54": 10.12837, + "55": 10.10246, + "56": 10.12757, + "57": 10.11585, + "58": 10.11966, + "59": 10.06692, + "60": 10.09295, + "61": 10.04272, + "62": 10.01401, + "63": 10.08132, + "64": 10.03233, + "65": 10.00041, + "66": 10.04317, + "67": 10.01559, + "68": 9.98468, + "69": 9.99673, "70": 9.97884, - "71": 10.00317, - "72": 9.99337, - "73": 9.97487, - "74": 9.96899, - "75": 9.93347, - "76": 9.96, - "77": 9.95963, - "78": 9.91408, - "79": 9.91446, - "80": 9.9278, - "81": 9.95364, - "82": 9.89077, - "83": 9.85458, - "84": 9.79534, - "85": 9.78077, - "86": 9.88652, - "87": 9.90728, - "88": 9.88177, - "89": 9.82134, - "90": 9.81596, - "91": 9.82611, - "92": 9.81644, - "93": 9.74789, - "94": 9.82692, - "95": 9.8124, - "96": 9.80068, - "97": 9.74582, - "98": 9.76759, - "99": 9.82081, - "100": 9.70464 + "71": 10.00321, + "72": 9.99341, + "73": 9.97492, + "74": 9.969, + "75": 9.93351, + "76": 9.96005, + "77": 9.95969, + "78": 9.91414, + "79": 9.91454, + "80": 9.92785, + "81": 9.95369, + "82": 9.89085, + "83": 9.85463, + "84": 9.79536, + "85": 9.78085, + "86": 9.88661, + "87": 9.9073, + "88": 9.8818, + "89": 9.82136, + "90": 9.816, + "91": 9.82615, + "92": 9.81647, + "93": 9.74796, + "94": 9.82703, + "95": 9.81241, + "96": 9.8007, + "97": 9.74586, + "98": 9.76766, + "99": 9.8209, + "100": 9.70466 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1860.0, - "2": 1723.0, - "3": 1744.0, - "4": 1706.0, - "5": 1728.0, - "6": 1819.0, - "7": 2115.0, - "8": 1842.0, - "9": 1780.0, - "10": 1833.0, - "11": 1729.0, - "12": 1700.0, - "13": 1796.0, - "14": 1892.0, - "15": 1685.0, - "16": 1830.0, - "17": 1776.0, - "18": 1808.0, - "19": 1773.0, - "20": 1677.0, - "21": 2010.0, - "22": 1730.0, - "23": 1757.0, - "24": 1871.0, - "25": 1729.0, - "26": 1948.0, - "27": 1828.0, - "28": 1936.0, - "29": 2089.0, - "30": 1939.0, - "31": 2035.0, - "32": 2110.0, - "33": 2069.0, - "34": 2052.0, - "35": 2140.0, - "36": 2138.0, - "37": 2316.0, - "38": 2220.0, - "39": 2331.0, - "40": 2410.0, - "41": 2521.0, - "42": 2051.0, - "43": 2356.0, - "44": 2270.0, - "45": 2659.0, - "46": 2383.0, - "47": 2670.0, - "48": 2604.0, - "49": 2861.0, - "50": 2560.0, - "51": 2702.0, - "52": 2710.0, - "53": 2608.0, - "54": 2882.0, - "55": 2566.0, - "56": 2846.0, - "57": 2382.0, - "58": 3522.0, - "59": 2982.0, - "60": 2976.0, - "61": 2778.0, - "62": 3235.0, - "63": 3319.0, - "64": 3725.0, - "65": 2682.0, - "66": 3128.0, - "67": 3676.0, - "68": 3436.0, - "69": 3089.0, - "70": 3343.0, - "71": 3411.0, - "72": 3064.0, - "73": 3578.0, - "74": 3498.0, - "75": 3220.0, - "76": 3470.0, - "77": 3781.0, - "78": 3435.0, - "79": 3325.0, - "80": 3168.0, - "81": 3600.0, - "82": 3084.0, - "83": 3292.0, - "84": 3021.0, - "85": 2762.0, - "86": 3196.0, - "87": 2781.0, - "88": 3162.0, - "89": 3267.0, - "90": 3876.0, - "91": 2952.0, - "92": 3211.0, - "93": 3445.0, - "94": 3375.0, - "95": 3328.0, - "96": 3457.0, - "97": 3632.0, - "98": 3494.0, - "99": 3069.0, - "100": 3360.0 + "1": 1897.0, + "2": 1693.0, + "3": 1785.0, + "4": 1691.0, + "5": 1765.0, + "6": 1832.0, + "7": 2153.0, + "8": 1834.0, + "9": 1751.0, + "10": 1791.0, + "11": 1778.0, + "12": 1680.0, + "13": 1806.0, + "14": 1915.0, + "15": 1720.0, + "16": 1841.0, + "17": 1811.0, + "18": 1776.0, + "19": 1723.0, + "20": 1726.0, + "21": 1832.0, + "22": 1713.0, + "23": 1723.0, + "24": 1819.0, + "25": 1703.0, + "26": 1877.0, + "27": 1806.0, + "28": 1938.0, + "29": 1995.0, + "30": 1919.0, + "31": 2005.0, + "32": 1965.0, + "33": 1994.0, + "34": 2076.0, + "35": 2122.0, + "36": 2117.0, + "37": 2351.0, + "38": 2115.0, + "39": 2246.0, + "40": 2414.0, + "41": 2365.0, + "42": 2069.0, + "43": 2425.0, + "44": 2293.0, + "45": 2609.0, + "46": 2531.0, + "47": 2522.0, + "48": 2574.0, + "49": 2867.0, + "50": 2583.0, + "51": 2634.0, + "52": 2795.0, + "53": 2595.0, + "54": 2824.0, + "55": 2624.0, + "56": 2765.0, + "57": 2296.0, + "58": 3650.0, + "59": 2935.0, + "60": 2986.0, + "61": 2742.0, + "62": 3163.0, + "63": 3254.0, + "64": 3652.0, + "65": 2725.0, + "66": 3111.0, + "67": 3787.0, + "68": 3519.0, + "69": 3026.0, + "70": 3419.0, + "71": 3449.0, + "72": 3011.0, + "73": 3577.0, + "74": 3419.0, + "75": 3242.0, + "76": 3559.0, + "77": 3865.0, + "78": 3384.0, + "79": 3317.0, + "80": 3183.0, + "81": 3667.0, + "82": 3210.0, + "83": 3425.0, + "84": 3026.0, + "85": 2825.0, + "86": 3139.0, + "87": 2908.0, + "88": 3234.0, + "89": 3210.0, + "90": 3866.0, + "91": 3020.0, + "92": 3238.0, + "93": 3394.0, + "94": 3383.0, + "95": 3258.0, + "96": 3419.0, + "97": 3694.0, + "98": 3517.0, + "99": 3076.0, + "100": 3275.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.44354, - "3": 0.1046, - "4": 0.10023, - "5": 0.10043, - "6": 3.02749, - "7": 0.5132, - "8": 1.15763, - "9": 0.90807, - "10": 1.94768, - "11": 0.10495, - "12": 0.10128, - "13": 0.09981, - "14": 0.71606, - "15": 1.59466, - "16": 0.10639, - "17": 1.80523, - "18": 1.2658, - "19": 0.30467, - "20": 0.47372, - "21": 0.10294, - "22": 2.03265, - "23": 0.10325, - "24": 0.09979, - "25": 0.53526, - "26": 0.10429, - "27": 1.60451, - "28": 0.11321, - "29": 0.68181, - "30": 0.10314, - "31": 1.54877, - "32": 0.27113, - "33": 1.29629, - "34": 0.87885, - "35": 0.10578, - "36": 0.31464, - "37": 0.98385, - "38": 0.1039, - "39": 0.94911, - "40": 0.10301, - "41": 1.47041, - "42": 0.31144, - "43": 0.6392, - "44": 0.11646, - "45": 1.05167, - "46": 0.11092, - "47": 1.51757, - "48": 0.19002, - "49": 1.24015, - "50": 0.10714, - "51": 0.1195, - "52": 0.1297, - "53": 0.10688, - "54": 0.10433, - "55": 1.40413, - "56": 0.78418, - "57": 0.53753, - "58": 0.10316, - "59": 1.71643, - "60": 0.10158, - "61": 0.53293, - "62": 0.10557, - "63": 0.70923, - "64": 0.14978, - "65": 0.52514, - "66": 1.56714, - "67": 0.10283, - "68": 1.07961, - "69": 0.34282, - "70": 0.54304, - "71": 0.10893, - "72": 0.25115, - "73": 0.48304, - "74": 0.22828, - "75": 0.65763, - "76": 0.21947, - "77": 1.34795, - "78": 0.10849, - "79": 0.68508, - "80": 0.51653, - "81": 0.51667, - "82": 0.25543, - "83": 0.37617, - "84": 0.60505, - "85": 0.44639, - "86": 0.90911, - "87": 0.53142, - "88": 0.34676, - "89": 0.76491, - "90": 0.97605, - "91": 0.10664, - "92": 1.10868, - "93": 0.10949, - "94": 0.10162, - "95": 0.10246, - "96": 0.10562, - "97": 0.68338, - "98": 1.0847, - "99": 0.62424, - "100": 0.27861 + "2": 5.06808, + "3": 0.10939, + "4": 0.10464, + "5": 0.10157, + "6": 3.19822, + "7": 0.10937, + "8": 1.95827, + "9": 0.10546, + "10": 2.40835, + "11": 0.10744, + "12": 0.24233, + "13": 0.10483, + "14": 1.60297, + "15": 0.43446, + "16": 1.12797, + "17": 0.50059, + "18": 2.0231, + "19": 0.10992, + "20": 0.52672, + "21": 0.29928, + "22": 1.32997, + "23": 0.8244, + "24": 0.10643, + "25": 1.06981, + "26": 0.10956, + "27": 1.10393, + "28": 0.12836, + "29": 0.54219, + "30": 0.12567, + "31": 1.27081, + "32": 0.42678, + "33": 1.17317, + "34": 0.10857, + "35": 0.61407, + "36": 1.19155, + "37": 0.7538, + "38": 0.1056, + "39": 1.40788, + "40": 0.10608, + "41": 2.37324, + "42": 0.30798, + "43": 0.9999, + "44": 0.10493, + "45": 1.50739, + "46": 0.11485, + "47": 1.71631, + "48": 0.10717, + "49": 1.34399, + "50": 0.10574, + "51": 0.11368, + "52": 0.12936, + "53": 1.2541, + "54": 0.28, + "55": 1.24609, + "56": 0.10819, + "57": 0.97754, + "58": 0.306, + "59": 1.7182, + "60": 0.10842, + "61": 0.73548, + "62": 0.10585, + "63": 1.88975, + "64": 0.13065, + "65": 1.57948, + "66": 0.38937, + "67": 0.10829, + "68": 1.05112, + "69": 1.16346, + "70": 0.50897, + "71": 0.10624, + "72": 0.35258, + "73": 0.96456, + "74": 1.18371, + "75": 0.15191, + "76": 0.99867, + "77": 1.17033, + "78": 1.01815, + "79": 0.10438, + "80": 1.13621, + "81": 0.1076, + "82": 0.48435, + "83": 1.02151, + "84": 0.10533, + "85": 0.98761, + "86": 0.60508, + "87": 1.26279, + "88": 0.1082, + "89": 1.33624, + "90": 0.10513, + "91": 0.90728, + "92": 0.34957, + "93": 0.61545, + "94": 0.32116, + "95": 0.61512, + "96": 0.65316, + "97": 0.21296, + "98": 1.48047, + "99": 0.45526, + "100": 0.37562 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json index 6b787dc554f..a74c33b10fc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.8851, - "2": 10.88271, - "3": 10.88387, - "4": 10.88623, - "5": 10.87916, - "6": 10.87661, + "1": 10.88508, + "2": 10.88267, + "3": 10.88381, + "4": 10.88628, + "5": 10.87917, + "6": 10.87655, "7": 10.87963, - "8": 10.87211, - "9": 10.88437, + "8": 10.87207, + "9": 10.88443, "10": 10.87858, - "11": 10.87354, - "12": 10.86326, - "13": 10.87879, - "14": 10.86595, - "15": 10.83782, - "16": 10.83261, - "17": 10.84044, - "18": 10.82306, - "19": 10.83569, + "11": 10.87347, + "12": 10.86336, + "13": 10.87875, + "14": 10.86592, + "15": 10.83783, + "16": 10.83257, + "17": 10.84046, + "18": 10.82313, + "19": 10.83571, "20": 10.72913, - "21": 10.7391, - "22": 10.72513, - "23": 10.71154, - "24": 10.68292, + "21": 10.73919, + "22": 10.72509, + "23": 10.71164, + "24": 10.68288, "25": 10.67538, - "26": 10.67981, - "27": 10.63493, - "28": 10.57986, + "26": 10.67985, + "27": 10.63489, + "28": 10.57994, "29": 10.54428, - "30": 10.5209, + "30": 10.52099, "31": 10.51407, - "32": 10.49437, - "33": 10.46826, - "34": 10.43401, - "35": 10.43851, - "36": 10.41311, - "37": 10.3791, - "38": 10.39376, - "39": 10.3513, - "40": 10.34595, - "41": 10.32313, - "42": 10.29756, - "43": 10.28542, - "44": 10.24461, - "45": 10.27146, - "46": 10.22651, - "47": 10.22023, - "48": 10.1733, - "49": 10.17078, - "50": 10.17217, - "51": 10.17766, - "52": 10.13083, - "53": 10.13975, - "54": 10.10412, - "55": 10.0701, - "56": 10.10628, - "57": 10.09819, - "58": 10.11158, - "59": 10.0548, - "60": 10.07314, - "61": 10.0271, - "62": 9.99675, + "32": 10.49436, + "33": 10.46832, + "34": 10.43403, + "35": 10.43854, + "36": 10.41312, + "37": 10.37911, + "38": 10.39378, + "39": 10.35137, + "40": 10.34605, + "41": 10.32321, + "42": 10.29759, + "43": 10.28549, + "44": 10.24468, + "45": 10.27149, + "46": 10.22659, + "47": 10.22032, + "48": 10.17336, + "49": 10.1708, + "50": 10.17222, + "51": 10.17772, + "52": 10.13088, + "53": 10.13981, + "54": 10.10415, + "55": 10.07012, + "56": 10.10633, + "57": 10.09818, + "58": 10.11161, + "59": 10.05485, + "60": 10.07321, + "61": 10.02715, + "62": 9.9968, "63": 10.07086, - "64": 10.02893, - "65": 10.00122, - "66": 10.02706, - "67": 10.00259, - "68": 9.96482, - "69": 9.9887, - "70": 9.97569, - "71": 9.99808, - "72": 9.97568, - "73": 9.96384, - "74": 9.95689, - "75": 9.92479, - "76": 9.9637, - "77": 9.95559, - "78": 9.90278, - "79": 9.90952, - "80": 9.92737, - "81": 9.94386, - "82": 9.89002, - "83": 9.84872, - "84": 9.78548, - "85": 9.77694, - "86": 9.87451, - "87": 9.90794, - "88": 9.88606, - "89": 9.81898, - "90": 9.81048, - "91": 9.82157, - "92": 9.8151, + "64": 10.02891, + "65": 10.00125, + "66": 10.02715, + "67": 10.00263, + "68": 9.96481, + "69": 9.98878, + "70": 9.97572, + "71": 9.99816, + "72": 9.97569, + "73": 9.96388, + "74": 9.95691, + "75": 9.92482, + "76": 9.96375, + "77": 9.95564, + "78": 9.90287, + "79": 9.90957, + "80": 9.92742, + "81": 9.9439, + "82": 9.89003, + "83": 9.84876, + "84": 9.78555, + "85": 9.77697, + "86": 9.87454, + "87": 9.908, + "88": 9.88615, + "89": 9.819, + "90": 9.81051, + "91": 9.82164, + "92": 9.81516, "93": 9.7511, - "94": 9.82585, - "95": 9.8205, - "96": 9.80342, - "97": 9.74065, - "98": 9.77415, - "99": 9.81989, - "100": 9.71431 + "94": 9.82591, + "95": 9.82053, + "96": 9.80346, + "97": 9.74068, + "98": 9.77419, + "99": 9.8199, + "100": 9.71434 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1709.0, - "2": 1738.0, - "3": 1755.0, - "4": 1816.0, - "5": 1698.0, - "6": 1631.0, - "7": 1869.0, - "8": 1739.0, - "9": 1691.0, - "10": 1826.0, - "11": 1722.0, - "12": 1730.0, - "13": 1818.0, - "14": 1863.0, - "15": 1615.0, - "16": 1836.0, - "17": 1747.0, - "18": 1833.0, - "19": 1608.0, - "20": 1657.0, - "21": 1859.0, - "22": 1730.0, - "23": 1635.0, - "24": 1832.0, - "25": 1729.0, - "26": 1798.0, - "27": 1878.0, - "28": 1847.0, - "29": 1947.0, - "30": 1779.0, - "31": 2074.0, - "32": 1982.0, - "33": 2070.0, - "34": 1922.0, - "35": 2024.0, - "36": 2020.0, - "37": 2265.0, - "38": 2093.0, - "39": 2265.0, - "40": 2284.0, - "41": 2386.0, - "42": 2091.0, - "43": 2397.0, - "44": 2272.0, - "45": 2554.0, - "46": 2392.0, - "47": 2509.0, - "48": 2705.0, - "49": 2865.0, - "50": 2674.0, - "51": 2545.0, - "52": 2776.0, - "53": 2583.0, - "54": 3041.0, - "55": 2759.0, - "56": 2815.0, - "57": 2403.0, - "58": 3632.0, - "59": 2959.0, - "60": 3070.0, - "61": 2800.0, - "62": 3300.0, - "63": 3522.0, - "64": 3671.0, - "65": 2744.0, - "66": 3233.0, - "67": 3924.0, - "68": 3752.0, - "69": 3066.0, - "70": 3516.0, - "71": 3069.0, - "72": 3063.0, - "73": 3411.0, - "74": 3374.0, - "75": 3206.0, - "76": 3251.0, - "77": 3808.0, - "78": 3312.0, - "79": 3270.0, - "80": 3004.0, - "81": 3497.0, - "82": 3130.0, - "83": 3054.0, - "84": 3133.0, - "85": 2843.0, - "86": 3235.0, - "87": 2937.0, - "88": 2995.0, - "89": 2953.0, - "90": 3492.0, - "91": 2867.0, - "92": 3067.0, - "93": 3202.0, - "94": 3100.0, - "95": 3381.0, - "96": 3427.0, - "97": 3692.0, - "98": 3619.0, - "99": 3378.0, - "100": 3170.0 + "1": 1824.0, + "2": 1713.0, + "3": 1872.0, + "4": 1690.0, + "5": 1686.0, + "6": 1613.0, + "7": 1828.0, + "8": 1740.0, + "9": 1761.0, + "10": 1718.0, + "11": 1674.0, + "12": 1721.0, + "13": 1855.0, + "14": 1832.0, + "15": 1623.0, + "16": 1675.0, + "17": 1829.0, + "18": 1877.0, + "19": 1661.0, + "20": 1668.0, + "21": 1791.0, + "22": 1691.0, + "23": 1676.0, + "24": 1798.0, + "25": 1659.0, + "26": 1858.0, + "27": 1851.0, + "28": 1863.0, + "29": 1878.0, + "30": 1877.0, + "31": 2045.0, + "32": 1918.0, + "33": 1977.0, + "34": 2058.0, + "35": 1992.0, + "36": 1963.0, + "37": 2267.0, + "38": 2114.0, + "39": 2218.0, + "40": 2258.0, + "41": 2353.0, + "42": 2072.0, + "43": 2406.0, + "44": 2273.0, + "45": 2683.0, + "46": 2461.0, + "47": 2463.0, + "48": 2636.0, + "49": 2904.0, + "50": 2602.0, + "51": 2624.0, + "52": 2797.0, + "53": 2568.0, + "54": 2935.0, + "55": 2728.0, + "56": 2948.0, + "57": 2298.0, + "58": 3679.0, + "59": 2956.0, + "60": 2922.0, + "61": 2807.0, + "62": 3241.0, + "63": 3504.0, + "64": 3682.0, + "65": 2727.0, + "66": 3156.0, + "67": 3950.0, + "68": 3649.0, + "69": 2946.0, + "70": 3482.0, + "71": 3226.0, + "72": 2982.0, + "73": 3523.0, + "74": 3385.0, + "75": 3168.0, + "76": 3320.0, + "77": 3701.0, + "78": 3334.0, + "79": 3194.0, + "80": 2976.0, + "81": 3505.0, + "82": 3045.0, + "83": 3097.0, + "84": 3067.0, + "85": 2814.0, + "86": 3264.0, + "87": 2959.0, + "88": 3047.0, + "89": 2921.0, + "90": 3453.0, + "91": 2971.0, + "92": 3022.0, + "93": 3243.0, + "94": 3117.0, + "95": 3551.0, + "96": 3367.0, + "97": 3815.0, + "98": 3809.0, + "99": 3355.0, + "100": 3176.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.28347, - "3": 0.20931, - "4": 0.82816, - "5": 0.42915, - "6": 0.68146, - "7": 0.58118, - "8": 0.99774, - "9": 0.61025, - "10": 0.35195, - "11": 0.97717, - "12": 0.3039, - "13": 0.64594, - "14": 0.64661, - "15": 0.38022, - "16": 0.873, - "17": 0.44623, - "18": 0.83025, - "19": 0.23953, - "20": 0.64854, - "21": 0.7005, - "22": 0.51888, - "23": 0.60687, - "24": 0.50759, - "25": 0.31193, - "26": 1.08874, - "27": 0.2125, - "28": 0.58008, - "29": 0.76487, - "30": 0.20537, - "31": 0.20034, - "32": 1.09629, - "33": 0.46348, - "34": 0.74156, - "35": 0.2114, - "36": 0.58307, - "37": 0.56521, - "38": 0.46852, - "39": 0.59962, - "40": 0.73748, - "41": 0.5621, - "42": 0.51396, - "43": 0.55903, - "44": 0.61807, - "45": 0.61512, - "46": 0.44588, - "47": 0.55796, - "48": 0.72719, - "49": 0.41084, - "50": 0.32797, - "51": 0.24383, - "52": 0.31474, - "53": 0.60589, - "54": 0.7403, - "55": 0.56813, - "56": 0.65665, - "57": 0.3942, - "58": 0.47181, - "59": 0.58245, - "60": 0.59921, - "61": 0.24895, - "62": 0.94832, - "63": 0.2419, - "64": 0.87643, - "65": 0.38117, - "66": 0.65093, - "67": 0.55319, - "68": 0.20829, - "69": 0.31956, - "70": 1.05681, - "71": 0.20846, - "72": 0.6042, - "73": 0.53133, - "74": 0.36375, - "75": 0.89679, - "76": 0.40789, - "77": 0.38711, - "78": 1.26522, - "79": 0.31147, - "80": 0.52379, - "81": 0.60742, - "82": 0.38747, - "83": 0.61588, - "84": 0.69754, - "85": 0.20763, - "86": 0.33517, - "87": 0.94717, - "88": 0.58336, - "89": 0.69931, - "90": 0.37694, - "91": 0.4405, - "92": 0.60648, - "93": 0.22284, - "94": 0.65936, - "95": 0.76711, - "96": 0.35046, - "97": 0.56031, - "98": 0.51784, - "99": 1.25475, - "100": 0.27365 + "2": 3.48372, + "3": 0.17215, + "4": 0.69128, + "5": 0.28783, + "6": 0.62889, + "7": 0.79182, + "8": 0.90226, + "9": 0.57106, + "10": 0.32674, + "11": 0.95774, + "12": 0.2999, + "13": 0.62001, + "14": 0.5513, + "15": 0.4462, + "16": 0.6675, + "17": 0.51446, + "18": 0.72015, + "19": 0.28585, + "20": 0.47276, + "21": 0.68088, + "22": 0.56058, + "23": 0.61689, + "24": 0.55035, + "25": 0.1788, + "26": 0.92946, + "27": 0.23837, + "28": 0.53631, + "29": 0.68202, + "30": 0.18322, + "31": 0.17731, + "32": 0.96592, + "33": 0.5926, + "34": 0.17439, + "35": 0.65416, + "36": 0.61808, + "37": 0.52055, + "38": 0.3845, + "39": 0.59178, + "40": 0.67875, + "41": 0.52473, + "42": 0.45164, + "43": 0.49765, + "44": 0.66679, + "45": 0.51455, + "46": 0.39138, + "47": 0.50501, + "48": 0.71761, + "49": 0.38153, + "50": 0.44859, + "51": 0.20052, + "52": 0.46988, + "53": 0.42244, + "54": 0.5206, + "55": 0.56679, + "56": 0.60752, + "57": 0.36001, + "58": 0.54577, + "59": 0.5098, + "60": 0.6557, + "61": 0.36124, + "62": 1.00677, + "63": 0.23972, + "64": 0.92097, + "65": 0.37023, + "66": 0.61255, + "67": 0.56812, + "68": 0.18103, + "69": 0.58984, + "70": 1.08323, + "71": 0.19369, + "72": 0.60572, + "73": 0.52119, + "74": 0.41819, + "75": 0.93187, + "76": 0.41853, + "77": 0.41466, + "78": 1.28969, + "79": 0.39952, + "80": 0.57404, + "81": 0.59192, + "82": 0.75006, + "83": 0.31663, + "84": 0.68383, + "85": 0.26024, + "86": 0.45461, + "87": 1.0516, + "88": 0.69715, + "89": 0.68941, + "90": 0.35455, + "91": 0.47533, + "92": 0.71535, + "93": 0.30525, + "94": 0.63106, + "95": 0.86963, + "96": 0.41378, + "97": 0.59183, + "98": 0.5809, + "99": 1.24167, + "100": 0.18879 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_fused/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_fused/model_config.yaml new file mode 100644 index 00000000000..13047bc9de1 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_fused/model_config.yaml @@ -0,0 +1,72 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + NCCL_ALGO: Ring + CUBLAS_WORKSPACE_CONFIG: :4096:8 + ENABLE_LIGHTWEIGHT_MODE: true +MODEL_ARGS: + --num-layers: 6 + --hidden-size: 512 + --num-attention-heads: 8 + --multi-latent-attention: true + --q-lora-rank: 192 + --kv-lora-rank: 64 + --qk-head-dim: 16 + --qk-pos-emb-head-dim: 8 + --v-head-dim: 16 + --experimental-attention-variant: dsv4_hybrid + --dsa-indexer-n-heads: 64 + --dsa-indexer-head-dim: 128 + --dsa-indexer-topk: 512 + --dsa-indexer-loss-coeff: 0.01 + --dsa-indexer-use-sparse-loss: true + --csa-window-size: 128 + --csa-compress-ratios: ([0,4,128,4,128,4]) + --csa-compress-rotary-base: 40000 + --attention-backend: fused + --pipeline-model-parallel-layout: "Et|tt|tt|tL" + --log-params-norm: true + --log-num-zeros-in-grad: true + --log-validation-ppl-to-tensorboard: true + --log-timers-to-tensorboard: true + --tensorboard-dir: ${TENSORBOARD_PATH} + --micro-batch-size: 4 + --global-batch-size: 32 + --seq-length: 1024 + --position-embedding-type: rope + --max-position-embeddings: 1024 + --train-iters: 50 + --timing-log-level: 0 + --lr-decay-iters: 320000 + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --split: 949,50,1 + --distributed-backend: nccl + --lr: 0.00015 + --lr-decay-style: cosine + --min-lr: 1.0e-5 + --weight-decay: 1e-2 + --clip-grad: 1.0 + --lr-warmup-fraction: .01 + --log-interval: 1 + --save-interval: 25 + --eval-interval: 1000 + --eval-iters: 10 + --transformer-impl: transformer_engine + --tensor-model-parallel-size: 1 + --pipeline-model-parallel-size: 2 + --sequence-parallel: true + --untie-embeddings-and-output-weights: true + --deterministic-mode: true + --no-gradient-accumulation-fusion: true + --attention-softmax-in-fp32: true + --use-mcore-models: true + --ckpt-format: torch_dist + --data-cache-path: ${DATA_CACHE_PATH} + --bf16: true + --attention-backend: unfused + --log-memory-to-tensorboard: true +TEST_TYPE: ckpt-resume diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml similarity index 98% rename from tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml rename to tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml index 6541e9d35cc..70efb998694 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_dsv4_hybrid_mhc_mtp/model_config.yaml @@ -23,6 +23,7 @@ MODEL_ARGS: --csa-window-size: 128 --csa-compress-ratios: ([0,4,128,4,0]) --csa-compress-rotary-base: 40000 + --no-dsa-kernel-fusion: true --attention-backend: fused --enable-hyper-connections: true --num-residual-streams: 4 diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json index ab55fa860d3..e1f8bae78cb 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89671, - "2": 10.90699, - "3": 10.91525, - "4": 10.9074, - "5": 10.90106, - "6": 10.88128, - "7": 10.89636, - "8": 10.90094, - "9": 10.90384, - "10": 10.90011, - "11": 10.88782, - "12": 10.88809, - "13": 10.87361, - "14": 10.87251, - "15": 10.86252, - "16": 10.84973, - "17": 10.85528, - "18": 10.83716, - "19": 10.83419, - "20": 10.75442, - "21": 10.75677, - "22": 10.74, - "23": 10.73105, - "24": 10.68857, - "25": 10.70425, + "1": 10.89667, + "2": 10.90696, + "3": 10.91523, + "4": 10.90739, + "5": 10.90111, + "6": 10.88125, + "7": 10.89639, + "8": 10.90091, + "9": 10.90387, + "10": 10.90015, + "11": 10.88791, + "12": 10.88821, + "13": 10.87366, + "14": 10.87253, + "15": 10.86264, + "16": 10.84971, + "17": 10.85529, + "18": 10.83714, + "19": 10.83418, + "20": 10.75447, + "21": 10.75681, + "22": 10.74009, + "23": 10.73108, + "24": 10.68855, + "25": 10.70426, "26": 10.67891, - "27": 10.64831, - "28": 10.56617, - "29": 10.54437, - "30": 10.51772, - "31": 10.51321, - "32": 10.48926, - "33": 10.46527, + "27": 10.64826, + "28": 10.56628, + "29": 10.5444, + "30": 10.51768, + "31": 10.51325, + "32": 10.48932, + "33": 10.46526, "34": 10.41453, - "35": 10.4193, - "36": 10.40481, - "37": 10.35419, - "38": 10.36334, - "39": 10.3237, - "40": 10.32131, - "41": 10.28361, + "35": 10.41938, + "36": 10.40485, + "37": 10.35425, + "38": 10.36339, + "39": 10.32379, + "40": 10.3213, + "41": 10.28367, "42": 10.27304, - "43": 10.24563, - "44": 10.21994, - "45": 10.23643, + "43": 10.24568, + "44": 10.21996, + "45": 10.23644, "46": 10.19726, - "47": 10.1835, - "48": 10.13263, - "49": 10.13315, + "47": 10.18346, + "48": 10.13267, + "49": 10.13323, "50": 10.14186, - "51": 10.14214, - "52": 10.09055, - "53": 10.09119, - "54": 10.06521, - "55": 10.01904, - "56": 10.0705, - "57": 10.0366, - "58": 10.0643, - "59": 10.00163, - "60": 10.01764, - "61": 9.98205, - "62": 9.93673, - "63": 10.03304, - "64": 9.971, - "65": 9.93122, - "66": 9.97348, - "67": 9.94526, - "68": 9.89723, - "69": 9.91389, - "70": 9.90381, - "71": 9.93242, - "72": 9.89127, - "73": 9.87283, - "74": 9.87351, - "75": 9.84003, - "76": 9.89839, - "77": 9.88911, - "78": 9.82531, - "79": 9.83361, - "80": 9.85186, - "81": 9.87648, - "82": 9.82806, - "83": 9.77263, - "84": 9.7092, - "85": 9.69224, - "86": 9.80612, - "87": 9.85432, - "88": 9.82674, - "89": 9.74654, - "90": 9.74087, - "91": 9.75407, - "92": 9.75061, - "93": 9.67293, + "51": 10.14218, + "52": 10.09057, + "53": 10.0912, + "54": 10.06522, + "55": 10.01914, + "56": 10.07052, + "57": 10.0367, + "58": 10.06435, + "59": 10.00168, + "60": 10.01769, + "61": 9.9821, + "62": 9.93679, + "63": 10.03311, + "64": 9.97102, + "65": 9.93128, + "66": 9.97356, + "67": 9.94529, + "68": 9.89729, + "69": 9.91388, + "70": 9.9039, + "71": 9.93248, + "72": 9.89128, + "73": 9.87288, + "74": 9.87353, + "75": 9.84011, + "76": 9.89845, + "77": 9.88917, + "78": 9.82537, + "79": 9.83364, + "80": 9.85189, + "81": 9.87652, + "82": 9.82812, + "83": 9.77279, + "84": 9.70928, + "85": 9.69232, + "86": 9.80609, + "87": 9.85435, + "88": 9.82679, + "89": 9.74657, + "90": 9.74097, + "91": 9.75411, + "92": 9.75063, + "93": 9.67299, "94": 9.75049, - "95": 9.75506, - "96": 9.73652, - "97": 9.65659, - "98": 9.7004, - "99": 9.75753, - "100": 9.63923 + "95": 9.75511, + "96": 9.73654, + "97": 9.65666, + "98": 9.70046, + "99": 9.75759, + "100": 9.63924 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2018.0, - "2": 1790.0, - "3": 2036.0, - "4": 1816.0, - "5": 2049.0, - "6": 1904.0, - "7": 2111.0, - "8": 1883.0, - "9": 1881.0, - "10": 1976.0, - "11": 1922.0, - "12": 1896.0, - "13": 2125.0, - "14": 2162.0, - "15": 1791.0, - "16": 1943.0, - "17": 2082.0, - "18": 1947.0, - "19": 1948.0, - "20": 1924.0, - "21": 2047.0, - "22": 1995.0, - "23": 1914.0, - "24": 2101.0, - "25": 1878.0, - "26": 1923.0, - "27": 1992.0, - "28": 1875.0, - "29": 1991.0, - "30": 1922.0, - "31": 2064.0, - "32": 1986.0, - "33": 1985.0, - "34": 2131.0, - "35": 2113.0, - "36": 2110.0, - "37": 2297.0, - "38": 2193.0, - "39": 2177.0, - "40": 2492.0, - "41": 2415.0, - "42": 2047.0, - "43": 2448.0, - "44": 2203.0, - "45": 2611.0, - "46": 2360.0, - "47": 2439.0, - "48": 2577.0, - "49": 2545.0, - "50": 2668.0, - "51": 2552.0, - "52": 2624.0, - "53": 2522.0, - "54": 2741.0, - "55": 2472.0, - "56": 2789.0, - "57": 2283.0, - "58": 3184.0, - "59": 2615.0, - "60": 2836.0, - "61": 2774.0, - "62": 2943.0, - "63": 3202.0, - "64": 3152.0, - "65": 2629.0, - "66": 2961.0, - "67": 3396.0, - "68": 3087.0, - "69": 3146.0, - "70": 2910.0, - "71": 2979.0, - "72": 2987.0, - "73": 3297.0, - "74": 3083.0, - "75": 3087.0, - "76": 3279.0, - "77": 3157.0, - "78": 3181.0, - "79": 2936.0, - "80": 2994.0, - "81": 3167.0, - "82": 3236.0, - "83": 2966.0, - "84": 2949.0, - "85": 2780.0, - "86": 3225.0, - "87": 3250.0, - "88": 3180.0, - "89": 3139.0, - "90": 3411.0, - "91": 2730.0, - "92": 2952.0, - "93": 3078.0, - "94": 3338.0, - "95": 3305.0, - "96": 3232.0, - "97": 3352.0, - "98": 3355.0, - "99": 3180.0, - "100": 3109.0 + "1": 2030.0, + "2": 1808.0, + "3": 2011.0, + "4": 1893.0, + "5": 1968.0, + "6": 1899.0, + "7": 2107.0, + "8": 1938.0, + "9": 1898.0, + "10": 1994.0, + "11": 1941.0, + "12": 1992.0, + "13": 2028.0, + "14": 2096.0, + "15": 1773.0, + "16": 1972.0, + "17": 1888.0, + "18": 2032.0, + "19": 1993.0, + "20": 1974.0, + "21": 1908.0, + "22": 1973.0, + "23": 1790.0, + "24": 2004.0, + "25": 1947.0, + "26": 1911.0, + "27": 2006.0, + "28": 1851.0, + "29": 2098.0, + "30": 1945.0, + "31": 2033.0, + "32": 1959.0, + "33": 2034.0, + "34": 2067.0, + "35": 2244.0, + "36": 2047.0, + "37": 2296.0, + "38": 2203.0, + "39": 2159.0, + "40": 2343.0, + "41": 2344.0, + "42": 2092.0, + "43": 2421.0, + "44": 2289.0, + "45": 2474.0, + "46": 2506.0, + "47": 2402.0, + "48": 2538.0, + "49": 2722.0, + "50": 2573.0, + "51": 2588.0, + "52": 2674.0, + "53": 2506.0, + "54": 2732.0, + "55": 2468.0, + "56": 2786.0, + "57": 2330.0, + "58": 3180.0, + "59": 2741.0, + "60": 2853.0, + "61": 2674.0, + "62": 3102.0, + "63": 3068.0, + "64": 3116.0, + "65": 2573.0, + "66": 2963.0, + "67": 3382.0, + "68": 3045.0, + "69": 3119.0, + "70": 2874.0, + "71": 3032.0, + "72": 2970.0, + "73": 3218.0, + "74": 3056.0, + "75": 3111.0, + "76": 3146.0, + "77": 3083.0, + "78": 3160.0, + "79": 2869.0, + "80": 2952.0, + "81": 3210.0, + "82": 3239.0, + "83": 2998.0, + "84": 2978.0, + "85": 2928.0, + "86": 3218.0, + "87": 3222.0, + "88": 3205.0, + "89": 3021.0, + "90": 3457.0, + "91": 2786.0, + "92": 2978.0, + "93": 2991.0, + "94": 3444.0, + "95": 3206.0, + "96": 3208.0, + "97": 3183.0, + "98": 3428.0, + "99": 3062.0, + "100": 3180.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 917459968.0, - "2": 917459968.0, - "3": 917459968.0, - "4": 917459968.0, - "5": 917459968.0, - "6": 917459968.0, - "7": 917459968.0, - "8": 917459968.0, - "9": 917459968.0, - "10": 917459968.0, - "11": 917459968.0, - "12": 917459968.0, - "13": 917459968.0, - "14": 917459968.0, - "15": 917459968.0, - "16": 917459968.0, - "17": 917459968.0, - "18": 917459968.0, - "19": 917459968.0, - "20": 917459968.0, - "21": 917459968.0, - "22": 917459968.0, - "23": 917459968.0, - "24": 917459968.0, - "25": 917459968.0, - "26": 917459968.0, - "27": 917459968.0, - "28": 917459968.0, - "29": 917459968.0, - "30": 917459968.0, - "31": 917459968.0, - "32": 917459968.0, - "33": 917459968.0, - "34": 917459968.0, - "35": 917459968.0, - "36": 917459968.0, - "37": 917459968.0, - "38": 917459968.0, - "39": 917459968.0, - "40": 917459968.0, - "41": 917459968.0, - "42": 917459968.0, - "43": 917459968.0, - "44": 917459968.0, - "45": 917459968.0, - "46": 917459968.0, - "47": 917459968.0, - "48": 917459968.0, - "49": 917459968.0, - "50": 917459968.0, - "51": 917459968.0, - "52": 917459968.0, - "53": 917459968.0, - "54": 917459968.0, - "55": 917459968.0, - "56": 917459968.0, - "57": 917459968.0, - "58": 917459968.0, - "59": 917459968.0, - "60": 917459968.0, - "61": 917459968.0, - "62": 917459968.0, - "63": 917459968.0, - "64": 917459968.0, - "65": 917459968.0, - "66": 917459968.0, - "67": 917459968.0, - "68": 917459968.0, - "69": 917459968.0, - "70": 917459968.0, - "71": 917459968.0, - "72": 917459968.0, - "73": 917459968.0, - "74": 917459968.0, - "75": 917459968.0, - "76": 917459968.0, - "77": 917459968.0, - "78": 917459968.0, - "79": 917459968.0, - "80": 917459968.0, - "81": 917459968.0, - "82": 917459968.0, - "83": 917459968.0, - "84": 917459968.0, - "85": 917459968.0, - "86": 917459968.0, - "87": 917459968.0, - "88": 917459968.0, - "89": 917459968.0, - "90": 917459968.0, - "91": 917459968.0, - "92": 917459968.0, - "93": 917459968.0, - "94": 917459968.0, - "95": 917459968.0, - "96": 917459968.0, - "97": 917459968.0, - "98": 917459968.0, - "99": 917459968.0, - "100": 917459968.0 + "1": 923751424.0, + "2": 923751424.0, + "3": 923751424.0, + "4": 923751424.0, + "5": 923751424.0, + "6": 923751424.0, + "7": 923751424.0, + "8": 923751424.0, + "9": 923751424.0, + "10": 923751424.0, + "11": 923751424.0, + "12": 923751424.0, + "13": 923751424.0, + "14": 923751424.0, + "15": 923751424.0, + "16": 923751424.0, + "17": 923751424.0, + "18": 923751424.0, + "19": 923751424.0, + "20": 923751424.0, + "21": 923751424.0, + "22": 923751424.0, + "23": 923751424.0, + "24": 923751424.0, + "25": 923751424.0, + "26": 923751424.0, + "27": 923751424.0, + "28": 923751424.0, + "29": 923751424.0, + "30": 923751424.0, + "31": 923751424.0, + "32": 923751424.0, + "33": 923751424.0, + "34": 923751424.0, + "35": 923751424.0, + "36": 923751424.0, + "37": 923751424.0, + "38": 923751424.0, + "39": 923751424.0, + "40": 923751424.0, + "41": 923751424.0, + "42": 923751424.0, + "43": 923751424.0, + "44": 923751424.0, + "45": 923751424.0, + "46": 923751424.0, + "47": 923751424.0, + "48": 923751424.0, + "49": 923751424.0, + "50": 923751424.0, + "51": 923751424.0, + "52": 923751424.0, + "53": 923751424.0, + "54": 923751424.0, + "55": 923751424.0, + "56": 923751424.0, + "57": 923751424.0, + "58": 923751424.0, + "59": 923751424.0, + "60": 923751424.0, + "61": 923751424.0, + "62": 923751424.0, + "63": 923751424.0, + "64": 923751424.0, + "65": 923751424.0, + "66": 923751424.0, + "67": 923751424.0, + "68": 923751424.0, + "69": 923751424.0, + "70": 923751424.0, + "71": 923751424.0, + "72": 923751424.0, + "73": 923751424.0, + "74": 923751424.0, + "75": 923751424.0, + "76": 923751424.0, + "77": 923751424.0, + "78": 923751424.0, + "79": 923751424.0, + "80": 923751424.0, + "81": 923751424.0, + "82": 923751424.0, + "83": 923751424.0, + "84": 923751424.0, + "85": 923751424.0, + "86": 923751424.0, + "87": 923751424.0, + "88": 923751424.0, + "89": 923751424.0, + "90": 923751424.0, + "91": 923751424.0, + "92": 923751424.0, + "93": 923751424.0, + "94": 923751424.0, + "95": 923751424.0, + "96": 923751424.0, + "97": 923751424.0, + "98": 923751424.0, + "99": 923751424.0, + "100": 923751424.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2262889472.0, - "2": 2621306880.0, - "3": 2621306880.0, - "4": 2621306880.0, - "5": 2621306880.0, - "6": 2621306880.0, - "7": 2621306880.0, - "8": 2621306880.0, - "9": 2621306880.0, - "10": 2621306880.0, - "11": 2621306880.0, - "12": 2621306880.0, - "13": 2621306880.0, - "14": 2621306880.0, - "15": 2621306880.0, - "16": 2621306880.0, - "17": 2621306880.0, - "18": 2621306880.0, - "19": 2621306880.0, - "20": 2621306880.0, - "21": 2621306880.0, - "22": 2621306880.0, - "23": 2621306880.0, - "24": 2621306880.0, - "25": 2621306880.0, - "26": 2621306880.0, - "27": 2621306880.0, - "28": 2621306880.0, - "29": 2621306880.0, - "30": 2621306880.0, - "31": 2621306880.0, - "32": 2621306880.0, - "33": 2621306880.0, - "34": 2621306880.0, - "35": 2621306880.0, - "36": 2621306880.0, - "37": 2621306880.0, - "38": 2621306880.0, - "39": 2621306880.0, - "40": 2621306880.0, - "41": 2621306880.0, - "42": 2621306880.0, - "43": 2621306880.0, - "44": 2621306880.0, - "45": 2621306880.0, - "46": 2621306880.0, - "47": 2621306880.0, - "48": 2621306880.0, - "49": 2621306880.0, - "50": 2621306880.0, - "51": 2621306880.0, - "52": 2621306880.0, - "53": 2621306880.0, - "54": 2621306880.0, - "55": 2621306880.0, - "56": 2621306880.0, - "57": 2621306880.0, - "58": 2621306880.0, - "59": 2621306880.0, - "60": 2621306880.0, - "61": 2621306880.0, - "62": 2621306880.0, - "63": 2621306880.0, - "64": 2621306880.0, - "65": 2621306880.0, - "66": 2621306880.0, - "67": 2621306880.0, - "68": 2621306880.0, - "69": 2621306880.0, - "70": 2621306880.0, - "71": 2621306880.0, - "72": 2621306880.0, - "73": 2621306880.0, - "74": 2621306880.0, - "75": 2621306880.0, - "76": 2621306880.0, - "77": 2621306880.0, - "78": 2621306880.0, - "79": 2621306880.0, - "80": 2621306880.0, - "81": 2621306880.0, - "82": 2621306880.0, - "83": 2621306880.0, - "84": 2621306880.0, - "85": 2621306880.0, - "86": 2621306880.0, - "87": 2621306880.0, - "88": 2621306880.0, - "89": 2621306880.0, - "90": 2621306880.0, - "91": 2621306880.0, - "92": 2621306880.0, - "93": 2621306880.0, - "94": 2621306880.0, - "95": 2621306880.0, - "96": 2621306880.0, - "97": 2621306880.0, - "98": 2621306880.0, - "99": 2621306880.0, - "100": 2621306880.0 + "2": 2627598336.0, + "3": 2627598336.0, + "4": 2627598336.0, + "5": 2627598336.0, + "6": 2627598336.0, + "7": 2627598336.0, + "8": 2627598336.0, + "9": 2627598336.0, + "10": 2627598336.0, + "11": 2627598336.0, + "12": 2627598336.0, + "13": 2627598336.0, + "14": 2627598336.0, + "15": 2627598336.0, + "16": 2627598336.0, + "17": 2627598336.0, + "18": 2627598336.0, + "19": 2627598336.0, + "20": 2627598336.0, + "21": 2627598336.0, + "22": 2627598336.0, + "23": 2627598336.0, + "24": 2627598336.0, + "25": 2627598336.0, + "26": 2627598336.0, + "27": 2627598336.0, + "28": 2627598336.0, + "29": 2627598336.0, + "30": 2627598336.0, + "31": 2627598336.0, + "32": 2627598336.0, + "33": 2627598336.0, + "34": 2627598336.0, + "35": 2627598336.0, + "36": 2627598336.0, + "37": 2627598336.0, + "38": 2627598336.0, + "39": 2627598336.0, + "40": 2627598336.0, + "41": 2627598336.0, + "42": 2627598336.0, + "43": 2627598336.0, + "44": 2627598336.0, + "45": 2627598336.0, + "46": 2627598336.0, + "47": 2627598336.0, + "48": 2627598336.0, + "49": 2627598336.0, + "50": 2627598336.0, + "51": 2627598336.0, + "52": 2627598336.0, + "53": 2627598336.0, + "54": 2627598336.0, + "55": 2627598336.0, + "56": 2627598336.0, + "57": 2627598336.0, + "58": 2627598336.0, + "59": 2627598336.0, + "60": 2627598336.0, + "61": 2627598336.0, + "62": 2627598336.0, + "63": 2627598336.0, + "64": 2627598336.0, + "65": 2627598336.0, + "66": 2627598336.0, + "67": 2627598336.0, + "68": 2627598336.0, + "69": 2627598336.0, + "70": 2627598336.0, + "71": 2627598336.0, + "72": 2627598336.0, + "73": 2627598336.0, + "74": 2627598336.0, + "75": 2627598336.0, + "76": 2627598336.0, + "77": 2627598336.0, + "78": 2627598336.0, + "79": 2627598336.0, + "80": 2627598336.0, + "81": 2627598336.0, + "82": 2627598336.0, + "83": 2627598336.0, + "84": 2627598336.0, + "85": 2627598336.0, + "86": 2627598336.0, + "87": 2627598336.0, + "88": 2627598336.0, + "89": 2627598336.0, + "90": 2627598336.0, + "91": 2627598336.0, + "92": 2627598336.0, + "93": 2627598336.0, + "94": 2627598336.0, + "95": 2627598336.0, + "96": 2627598336.0, + "97": 2627598336.0, + "98": 2627598336.0, + "99": 2627598336.0, + "100": 2627598336.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.45285, - "3": 0.1436, - "4": 0.12921, - "5": 0.12859, - "6": 0.12814, - "7": 0.12748, - "8": 0.12785, - "9": 0.12919, - "10": 0.12768, - "11": 0.13037, - "12": 0.12931, - "13": 0.12844, - "14": 0.12922, - "15": 0.13009, - "16": 0.13005, - "17": 0.12939, - "18": 0.13026, - "19": 0.12967, - "20": 0.1289, - "21": 0.12943, - "22": 0.12844, - "23": 0.1287, - "24": 0.12905, - "25": 0.12825, - "26": 0.13045, - "27": 0.12925, - "28": 0.12893, - "29": 0.12927, - "30": 0.12828, - "31": 0.12861, - "32": 0.12927, - "33": 0.13057, - "34": 0.1291, - "35": 0.12916, - "36": 0.12831, - "37": 0.12897, - "38": 0.12902, - "39": 0.12976, - "40": 0.12893, - "41": 0.12897, - "42": 0.12925, - "43": 0.12979, - "44": 0.13026, - "45": 0.12975, - "46": 0.12974, - "47": 0.12991, - "48": 0.12959, - "49": 0.12962, - "50": 0.1296, - "51": 0.28365, - "52": 0.13758, - "53": 0.16001, - "54": 0.12881, - "55": 0.12958, - "56": 0.12825, - "57": 0.12896, - "58": 0.12818, - "59": 0.12767, - "60": 0.12796, - "61": 0.1285, - "62": 0.12727, - "63": 0.12736, - "64": 0.12712, - "65": 0.12715, - "66": 0.12769, - "67": 0.12855, - "68": 0.12709, - "69": 0.12705, - "70": 0.12647, - "71": 0.12756, - "72": 0.12588, - "73": 0.12675, - "74": 0.12724, - "75": 0.12721, - "76": 0.1277, - "77": 0.12816, - "78": 0.12766, - "79": 0.12758, - "80": 0.12747, - "81": 0.12819, - "82": 0.12763, - "83": 0.12863, - "84": 0.12887, - "85": 0.12982, - "86": 0.12788, - "87": 0.12801, - "88": 0.12708, - "89": 0.12701, - "90": 0.12771, - "91": 0.12778, - "92": 0.12793, - "93": 0.12752, - "94": 0.12745, - "95": 0.12745, - "96": 0.12758, - "97": 0.12766, - "98": 0.12817, - "99": 0.12833, - "100": 0.12813 + "2": 6.96038, + "3": 0.14559, + "4": 0.1053, + "5": 0.10472, + "6": 0.10556, + "7": 0.10637, + "8": 0.10577, + "9": 0.10703, + "10": 0.1047, + "11": 0.10499, + "12": 0.10411, + "13": 0.10523, + "14": 0.10465, + "15": 0.10424, + "16": 0.10501, + "17": 0.10439, + "18": 0.10606, + "19": 0.10508, + "20": 0.10312, + "21": 0.10418, + "22": 0.10443, + "23": 0.1039, + "24": 0.10427, + "25": 0.10382, + "26": 0.10566, + "27": 0.10697, + "28": 0.10526, + "29": 0.10349, + "30": 0.10412, + "31": 0.10461, + "32": 0.10383, + "33": 0.10521, + "34": 0.10488, + "35": 0.10442, + "36": 0.10481, + "37": 0.10326, + "38": 0.1044, + "39": 0.10389, + "40": 0.10538, + "41": 0.1044, + "42": 0.13653, + "43": 0.10462, + "44": 0.10453, + "45": 0.10668, + "46": 0.10432, + "47": 0.10508, + "48": 0.10369, + "49": 0.10494, + "50": 0.10518, + "51": 0.27989, + "52": 0.11884, + "53": 0.14653, + "54": 0.10914, + "55": 0.10764, + "56": 0.10542, + "57": 0.10517, + "58": 0.1084, + "59": 0.10642, + "60": 0.10543, + "61": 0.10638, + "62": 0.10579, + "63": 0.10363, + "64": 0.105, + "65": 0.10464, + "66": 0.10483, + "67": 0.10555, + "68": 0.10515, + "69": 0.10464, + "70": 0.10537, + "71": 0.10596, + "72": 0.10365, + "73": 0.13042, + "74": 0.10536, + "75": 0.10717, + "76": 0.10613, + "77": 0.12067, + "78": 0.10457, + "79": 0.10572, + "80": 0.10409, + "81": 0.1053, + "82": 0.10593, + "83": 0.10661, + "84": 0.10554, + "85": 0.10556, + "86": 0.10526, + "87": 0.10618, + "88": 0.1062, + "89": 0.1058, + "90": 0.10593, + "91": 0.1068, + "92": 0.10472, + "93": 0.10524, + "94": 0.10746, + "95": 0.10595, + "96": 0.10585, + "97": 0.10544, + "98": 0.10716, + "99": 0.10565, + "100": 0.10484 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json index 2cca7545f14..85ef64219fb 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96096, + "1": 10.96095, "2": 10.94378, - "3": 10.93765, - "4": 10.95048, - "5": 10.9484, - "6": 10.94213, - "7": 10.93466, - "8": 10.9418, - "9": 10.94159, - "10": 10.9279, - "11": 10.93407, - "12": 10.94464, - "13": 10.92188, - "14": 10.92239, - "15": 10.89966, - "16": 10.89036, - "17": 10.88908, + "3": 10.9376, + "4": 10.95053, + "5": 10.94832, + "6": 10.94216, + "7": 10.93458, + "8": 10.94185, + "9": 10.94157, + "10": 10.92794, + "11": 10.93401, + "12": 10.94467, + "13": 10.92192, + "14": 10.92242, + "15": 10.89964, + "16": 10.8904, + "17": 10.88905, "18": 10.8818, - "19": 10.87644, - "20": 10.80072, - "21": 10.78156, - "22": 10.76187, - "23": 10.76245, - "24": 10.73665, - "25": 10.73432, - "26": 10.72099, - "27": 10.68134, - "28": 10.59567, - "29": 10.57425, - "30": 10.55166, - "31": 10.55268, - "32": 10.52959, + "19": 10.87634, + "20": 10.80079, + "21": 10.78165, + "22": 10.76191, + "23": 10.76255, + "24": 10.73668, + "25": 10.73435, + "26": 10.72101, + "27": 10.6814, + "28": 10.59571, + "29": 10.5743, + "30": 10.5518, + "31": 10.55277, + "32": 10.52963, "33": 10.49301, - "34": 10.46574, - "35": 10.46781, - "36": 10.4343, - "37": 10.40562, - "38": 10.40708, - "39": 10.38088, - "40": 10.35374, - "41": 10.33103, - "42": 10.31722, - "43": 10.27722, - "44": 10.27454, + "34": 10.46588, + "35": 10.46778, + "36": 10.43436, + "37": 10.40571, + "38": 10.4071, + "39": 10.3809, + "40": 10.35377, + "41": 10.3311, + "42": 10.3172, + "43": 10.27725, + "44": 10.27456, "45": 10.27064, - "46": 10.23923, - "47": 10.22013, - "48": 10.17386, - "49": 10.18283, - "50": 10.18243, - "51": 10.17726, - "52": 10.12542, - "53": 10.11928, - "54": 10.09855, - "55": 10.06505, + "46": 10.23931, + "47": 10.22016, + "48": 10.17388, + "49": 10.18285, + "50": 10.18249, + "51": 10.1773, + "52": 10.12547, + "53": 10.11934, + "54": 10.0986, + "55": 10.06513, "56": 10.096, - "57": 10.07028, - "58": 10.08099, - "59": 10.02378, - "60": 10.04446, - "61": 10.00535, - "62": 9.96184, - "63": 10.04441, - "64": 9.98732, - "65": 9.94123, - "66": 9.99568, - "67": 9.96328, - "68": 9.92304, - "69": 9.92538, - "70": 9.91447, - "71": 9.94788, - "72": 9.91509, + "57": 10.07031, + "58": 10.08104, + "59": 10.02385, + "60": 10.04452, + "61": 10.00537, + "62": 9.96197, + "63": 10.04444, + "64": 9.98734, + "65": 9.94133, + "66": 9.99575, + "67": 9.9634, + "68": 9.9231, + "69": 9.92541, + "70": 9.91453, + "71": 9.94791, + "72": 9.91515, "73": 9.89097, - "74": 9.8886, - "75": 9.84715, - "76": 9.8959, - "77": 9.89051, - "78": 9.84141, - "79": 9.83907, - "80": 9.85622, - "81": 9.88218, - "82": 9.83193, - "83": 9.77675, - "84": 9.71176, - "85": 9.69855, - "86": 9.81118, - "87": 9.84717, - "88": 9.81948, - "89": 9.74334, - "90": 9.74338, - "91": 9.75546, - "92": 9.75109, - "93": 9.66382, - "94": 9.7469, - "95": 9.74363, - "96": 9.7211, - "97": 9.65221, - "98": 9.68133, - "99": 9.75593, - "100": 9.62787 + "74": 9.88869, + "75": 9.84723, + "76": 9.89598, + "77": 9.89061, + "78": 9.84146, + "79": 9.83915, + "80": 9.85629, + "81": 9.88221, + "82": 9.83194, + "83": 9.77679, + "84": 9.71183, + "85": 9.69863, + "86": 9.81127, + "87": 9.84721, + "88": 9.81959, + "89": 9.74338, + "90": 9.74347, + "91": 9.75549, + "92": 9.75117, + "93": 9.66392, + "94": 9.74692, + "95": 9.7437, + "96": 9.72111, + "97": 9.65225, + "98": 9.68137, + "99": 9.75599, + "100": 9.62793 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1958.0, - "2": 1885.0, - "3": 1969.0, - "4": 1999.0, - "5": 1867.0, - "6": 1911.0, - "7": 2177.0, - "8": 1891.0, - "9": 1929.0, - "10": 1805.0, - "11": 1962.0, - "12": 1807.0, - "13": 2025.0, - "14": 2027.0, - "15": 1879.0, - "16": 1843.0, - "17": 1905.0, - "18": 1951.0, - "19": 1818.0, - "20": 1855.0, - "21": 2073.0, - "22": 1999.0, - "23": 1931.0, - "24": 1954.0, - "25": 1885.0, - "26": 1873.0, - "27": 1933.0, - "28": 1849.0, - "29": 1846.0, - "30": 1938.0, - "31": 2100.0, - "32": 1916.0, - "33": 1980.0, + "1": 1892.0, + "2": 1813.0, + "3": 1829.0, + "4": 1973.0, + "5": 1882.0, + "6": 2001.0, + "7": 2094.0, + "8": 1871.0, + "9": 1941.0, + "10": 1883.0, + "11": 1940.0, + "12": 1831.0, + "13": 1993.0, + "14": 1894.0, + "15": 1719.0, + "16": 1863.0, + "17": 1892.0, + "18": 1879.0, + "19": 1863.0, + "20": 1794.0, + "21": 2053.0, + "22": 1956.0, + "23": 1953.0, + "24": 2021.0, + "25": 1899.0, + "26": 2001.0, + "27": 2069.0, + "28": 1873.0, + "29": 1915.0, + "30": 1820.0, + "31": 2063.0, + "32": 1895.0, + "33": 1945.0, "34": 2037.0, - "35": 2186.0, - "36": 2150.0, - "37": 2263.0, - "38": 2156.0, - "39": 2189.0, - "40": 2404.0, - "41": 2412.0, - "42": 2103.0, - "43": 2402.0, - "44": 2177.0, - "45": 2409.0, - "46": 2603.0, - "47": 2426.0, - "48": 2473.0, - "49": 2667.0, - "50": 2655.0, - "51": 2533.0, - "52": 2515.0, - "53": 2545.0, - "54": 2733.0, - "55": 2493.0, - "56": 2587.0, - "57": 2363.0, - "58": 3251.0, - "59": 2811.0, - "60": 2742.0, - "61": 2684.0, - "62": 3000.0, - "63": 3044.0, - "64": 3028.0, - "65": 2515.0, - "66": 2860.0, - "67": 3289.0, - "68": 2968.0, - "69": 2906.0, - "70": 2961.0, - "71": 2931.0, - "72": 3076.0, - "73": 3177.0, - "74": 3036.0, - "75": 2891.0, - "76": 3299.0, - "77": 3251.0, - "78": 2998.0, - "79": 2895.0, - "80": 3002.0, - "81": 3079.0, - "82": 3229.0, - "83": 2912.0, - "84": 2982.0, - "85": 2899.0, - "86": 3206.0, - "87": 2993.0, - "88": 3126.0, - "89": 3111.0, - "90": 3471.0, - "91": 2851.0, - "92": 2966.0, - "93": 2875.0, - "94": 3261.0, - "95": 3139.0, - "96": 3289.0, - "97": 3318.0, - "98": 3249.0, - "99": 3044.0, - "100": 2947.0 + "35": 2099.0, + "36": 2164.0, + "37": 2249.0, + "38": 2197.0, + "39": 2095.0, + "40": 2400.0, + "41": 2421.0, + "42": 2180.0, + "43": 2362.0, + "44": 2254.0, + "45": 2458.0, + "46": 2484.0, + "47": 2365.0, + "48": 2506.0, + "49": 2664.0, + "50": 2618.0, + "51": 2408.0, + "52": 2520.0, + "53": 2461.0, + "54": 2709.0, + "55": 2468.0, + "56": 2650.0, + "57": 2388.0, + "58": 3177.0, + "59": 2817.0, + "60": 2841.0, + "61": 2659.0, + "62": 2852.0, + "63": 3055.0, + "64": 3053.0, + "65": 2545.0, + "66": 2937.0, + "67": 3209.0, + "68": 2972.0, + "69": 2964.0, + "70": 2833.0, + "71": 2937.0, + "72": 3097.0, + "73": 3281.0, + "74": 2983.0, + "75": 2940.0, + "76": 3261.0, + "77": 3204.0, + "78": 2954.0, + "79": 2999.0, + "80": 2958.0, + "81": 3106.0, + "82": 3079.0, + "83": 2905.0, + "84": 2938.0, + "85": 3005.0, + "86": 3324.0, + "87": 2949.0, + "88": 3234.0, + "89": 3061.0, + "90": 3457.0, + "91": 2908.0, + "92": 3059.0, + "93": 2999.0, + "94": 3250.0, + "95": 3091.0, + "96": 3211.0, + "97": 3342.0, + "98": 3247.0, + "99": 3091.0, + "100": 2944.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 919557120.0, - "2": 919557120.0, - "3": 919557120.0, - "4": 919557120.0, - "5": 919557120.0, - "6": 919557120.0, - "7": 919557120.0, - "8": 919557120.0, - "9": 919557120.0, - "10": 919557120.0, - "11": 919557120.0, - "12": 919557120.0, - "13": 919557120.0, - "14": 919557120.0, - "15": 919557120.0, - "16": 919557120.0, - "17": 919557120.0, - "18": 919557120.0, - "19": 919557120.0, - "20": 919557120.0, - "21": 919557120.0, - "22": 919557120.0, - "23": 919557120.0, - "24": 919557120.0, - "25": 919557120.0, - "26": 919557120.0, - "27": 919557120.0, - "28": 919557120.0, - "29": 919557120.0, - "30": 919557120.0, - "31": 919557120.0, - "32": 919557120.0, - "33": 919557120.0, - "34": 919557120.0, - "35": 919557120.0, - "36": 919557120.0, - "37": 919557120.0, - "38": 919557120.0, - "39": 919557120.0, - "40": 919557120.0, - "41": 919557120.0, - "42": 919557120.0, - "43": 919557120.0, - "44": 919557120.0, - "45": 919557120.0, - "46": 919557120.0, - "47": 919557120.0, - "48": 919557120.0, - "49": 919557120.0, - "50": 919557120.0, - "51": 919557120.0, - "52": 919557120.0, - "53": 919557120.0, - "54": 919557120.0, - "55": 919557120.0, - "56": 919557120.0, - "57": 919557120.0, - "58": 919557120.0, - "59": 919557120.0, - "60": 919557120.0, - "61": 919557120.0, - "62": 919557120.0, - "63": 919557120.0, - "64": 919557120.0, - "65": 919557120.0, - "66": 919557120.0, - "67": 919557120.0, - "68": 919557120.0, - "69": 919557120.0, - "70": 919557120.0, - "71": 919557120.0, - "72": 919557120.0, - "73": 919557120.0, - "74": 919557120.0, - "75": 919557120.0, - "76": 919557120.0, - "77": 919557120.0, - "78": 919557120.0, - "79": 919557120.0, - "80": 919557120.0, - "81": 919557120.0, - "82": 919557120.0, - "83": 919557120.0, - "84": 919557120.0, - "85": 919557120.0, - "86": 919557120.0, - "87": 919557120.0, - "88": 919557120.0, - "89": 919557120.0, - "90": 919557120.0, - "91": 919557120.0, - "92": 919557120.0, - "93": 919557120.0, - "94": 919557120.0, - "95": 919557120.0, - "96": 919557120.0, - "97": 919557120.0, - "98": 919557120.0, - "99": 919557120.0, - "100": 919557120.0 + "1": 917459968.0, + "2": 917459968.0, + "3": 917459968.0, + "4": 917459968.0, + "5": 917459968.0, + "6": 917459968.0, + "7": 917459968.0, + "8": 917459968.0, + "9": 917459968.0, + "10": 917459968.0, + "11": 917459968.0, + "12": 917459968.0, + "13": 917459968.0, + "14": 917459968.0, + "15": 917459968.0, + "16": 917459968.0, + "17": 917459968.0, + "18": 917459968.0, + "19": 917459968.0, + "20": 917459968.0, + "21": 917459968.0, + "22": 917459968.0, + "23": 917459968.0, + "24": 917459968.0, + "25": 917459968.0, + "26": 917459968.0, + "27": 917459968.0, + "28": 917459968.0, + "29": 917459968.0, + "30": 917459968.0, + "31": 917459968.0, + "32": 917459968.0, + "33": 917459968.0, + "34": 917459968.0, + "35": 917459968.0, + "36": 917459968.0, + "37": 917459968.0, + "38": 917459968.0, + "39": 917459968.0, + "40": 917459968.0, + "41": 917459968.0, + "42": 917459968.0, + "43": 917459968.0, + "44": 917459968.0, + "45": 917459968.0, + "46": 917459968.0, + "47": 917459968.0, + "48": 917459968.0, + "49": 917459968.0, + "50": 917459968.0, + "51": 917459968.0, + "52": 917459968.0, + "53": 917459968.0, + "54": 917459968.0, + "55": 917459968.0, + "56": 917459968.0, + "57": 917459968.0, + "58": 917459968.0, + "59": 917459968.0, + "60": 917459968.0, + "61": 917459968.0, + "62": 917459968.0, + "63": 917459968.0, + "64": 917459968.0, + "65": 917459968.0, + "66": 917459968.0, + "67": 917459968.0, + "68": 917459968.0, + "69": 917459968.0, + "70": 917459968.0, + "71": 917459968.0, + "72": 917459968.0, + "73": 917459968.0, + "74": 917459968.0, + "75": 917459968.0, + "76": 917459968.0, + "77": 917459968.0, + "78": 917459968.0, + "79": 917459968.0, + "80": 917459968.0, + "81": 917459968.0, + "82": 917459968.0, + "83": 917459968.0, + "84": 917459968.0, + "85": 917459968.0, + "86": 917459968.0, + "87": 917459968.0, + "88": 917459968.0, + "89": 917459968.0, + "90": 917459968.0, + "91": 917459968.0, + "92": 917459968.0, + "93": 917459968.0, + "94": 917459968.0, + "95": 917459968.0, + "96": 917459968.0, + "97": 917459968.0, + "98": 917459968.0, + "99": 917459968.0, + "100": 917459968.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2237723648.0, - "2": 2598238208.0, - "3": 2598238208.0, - "4": 2598238208.0, - "5": 2598238208.0, - "6": 2598238208.0, - "7": 2598238208.0, - "8": 2598238208.0, - "9": 2598238208.0, - "10": 2598238208.0, - "11": 2598238208.0, - "12": 2598238208.0, - "13": 2598238208.0, - "14": 2598238208.0, - "15": 2598238208.0, - "16": 2598238208.0, - "17": 2598238208.0, - "18": 2598238208.0, - "19": 2598238208.0, - "20": 2598238208.0, - "21": 2598238208.0, - "22": 2598238208.0, - "23": 2598238208.0, - "24": 2598238208.0, - "25": 2598238208.0, - "26": 2598238208.0, - "27": 2598238208.0, - "28": 2598238208.0, - "29": 2598238208.0, - "30": 2598238208.0, - "31": 2598238208.0, - "32": 2598238208.0, - "33": 2598238208.0, - "34": 2598238208.0, - "35": 2598238208.0, - "36": 2598238208.0, - "37": 2598238208.0, - "38": 2598238208.0, - "39": 2598238208.0, - "40": 2598238208.0, - "41": 2598238208.0, - "42": 2598238208.0, - "43": 2598238208.0, - "44": 2598238208.0, - "45": 2598238208.0, - "46": 2598238208.0, - "47": 2598238208.0, - "48": 2598238208.0, - "49": 2598238208.0, - "50": 2598238208.0, - "51": 2598238208.0, - "52": 2598238208.0, - "53": 2598238208.0, - "54": 2598238208.0, - "55": 2598238208.0, - "56": 2598238208.0, - "57": 2598238208.0, - "58": 2598238208.0, - "59": 2598238208.0, - "60": 2598238208.0, - "61": 2598238208.0, - "62": 2598238208.0, - "63": 2598238208.0, - "64": 2598238208.0, - "65": 2598238208.0, - "66": 2598238208.0, - "67": 2598238208.0, - "68": 2598238208.0, - "69": 2598238208.0, - "70": 2598238208.0, - "71": 2598238208.0, - "72": 2598238208.0, - "73": 2598238208.0, - "74": 2598238208.0, - "75": 2598238208.0, - "76": 2598238208.0, - "77": 2598238208.0, - "78": 2598238208.0, - "79": 2598238208.0, - "80": 2598238208.0, - "81": 2598238208.0, - "82": 2598238208.0, - "83": 2598238208.0, - "84": 2598238208.0, - "85": 2598238208.0, - "86": 2598238208.0, - "87": 2598238208.0, - "88": 2598238208.0, - "89": 2598238208.0, - "90": 2598238208.0, - "91": 2598238208.0, - "92": 2598238208.0, - "93": 2598238208.0, - "94": 2598238208.0, - "95": 2598238208.0, - "96": 2598238208.0, - "97": 2598238208.0, - "98": 2598238208.0, - "99": 2598238208.0, - "100": 2598238208.0 + "2": 2596141056.0, + "3": 2596141056.0, + "4": 2596141056.0, + "5": 2596141056.0, + "6": 2596141056.0, + "7": 2596141056.0, + "8": 2596141056.0, + "9": 2596141056.0, + "10": 2596141056.0, + "11": 2596141056.0, + "12": 2596141056.0, + "13": 2596141056.0, + "14": 2596141056.0, + "15": 2596141056.0, + "16": 2596141056.0, + "17": 2596141056.0, + "18": 2596141056.0, + "19": 2596141056.0, + "20": 2596141056.0, + "21": 2596141056.0, + "22": 2596141056.0, + "23": 2596141056.0, + "24": 2596141056.0, + "25": 2596141056.0, + "26": 2596141056.0, + "27": 2596141056.0, + "28": 2596141056.0, + "29": 2596141056.0, + "30": 2596141056.0, + "31": 2596141056.0, + "32": 2596141056.0, + "33": 2596141056.0, + "34": 2596141056.0, + "35": 2596141056.0, + "36": 2596141056.0, + "37": 2596141056.0, + "38": 2596141056.0, + "39": 2596141056.0, + "40": 2596141056.0, + "41": 2596141056.0, + "42": 2596141056.0, + "43": 2596141056.0, + "44": 2596141056.0, + "45": 2596141056.0, + "46": 2596141056.0, + "47": 2596141056.0, + "48": 2596141056.0, + "49": 2596141056.0, + "50": 2596141056.0, + "51": 2596141056.0, + "52": 2596141056.0, + "53": 2596141056.0, + "54": 2596141056.0, + "55": 2596141056.0, + "56": 2596141056.0, + "57": 2596141056.0, + "58": 2596141056.0, + "59": 2596141056.0, + "60": 2596141056.0, + "61": 2596141056.0, + "62": 2596141056.0, + "63": 2596141056.0, + "64": 2596141056.0, + "65": 2596141056.0, + "66": 2596141056.0, + "67": 2596141056.0, + "68": 2596141056.0, + "69": 2596141056.0, + "70": 2596141056.0, + "71": 2596141056.0, + "72": 2596141056.0, + "73": 2596141056.0, + "74": 2596141056.0, + "75": 2596141056.0, + "76": 2596141056.0, + "77": 2596141056.0, + "78": 2596141056.0, + "79": 2596141056.0, + "80": 2596141056.0, + "81": 2596141056.0, + "82": 2596141056.0, + "83": 2596141056.0, + "84": 2596141056.0, + "85": 2596141056.0, + "86": 2596141056.0, + "87": 2596141056.0, + "88": 2596141056.0, + "89": 2596141056.0, + "90": 2596141056.0, + "91": 2596141056.0, + "92": 2596141056.0, + "93": 2596141056.0, + "94": 2596141056.0, + "95": 2596141056.0, + "96": 2596141056.0, + "97": 2596141056.0, + "98": 2596141056.0, + "99": 2596141056.0, + "100": 2596141056.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.48074, - "3": 0.11631, - "4": 0.11636, - "5": 0.11519, - "6": 0.11496, - "7": 0.11617, - "8": 0.11552, - "9": 0.11484, - "10": 0.11517, - "11": 0.11614, - "12": 0.11662, - "13": 0.11445, - "14": 0.11535, - "15": 0.11445, - "16": 0.11409, - "17": 0.11458, - "18": 0.11383, - "19": 0.11873, - "20": 0.11503, - "21": 0.11544, - "22": 0.1154, - "23": 0.1164, - "24": 0.12656, - "25": 0.11763, - "26": 0.11572, - "27": 0.11553, - "28": 0.1148, - "29": 0.11577, - "30": 0.11507, - "31": 0.11538, - "32": 0.11684, - "33": 0.1163, - "34": 0.1153, - "35": 0.1148, - "36": 0.11434, - "37": 0.11776, - "38": 0.11635, - "39": 0.11641, - "40": 0.1155, - "41": 0.1141, - "42": 0.11503, - "43": 0.11715, - "44": 0.11489, - "45": 0.11544, - "46": 0.11474, - "47": 0.11803, - "48": 0.11512, - "49": 0.11465, - "50": 0.11459, - "51": 0.14805, - "52": 0.1391, - "53": 0.12179, - "54": 0.12138, - "55": 0.11726, - "56": 0.11668, - "57": 0.11745, - "58": 0.11896, - "59": 0.11676, - "60": 0.11642, - "61": 0.11523, - "62": 0.11424, - "63": 0.11435, - "64": 0.11653, - "65": 0.11928, - "66": 0.11731, - "67": 0.11703, - "68": 0.11679, - "69": 0.11634, - "70": 0.11642, - "71": 0.11527, - "72": 0.11515, - "73": 0.1164, - "74": 0.11599, - "75": 0.11673, - "76": 0.11705, - "77": 0.11642, - "78": 0.11446, - "79": 0.11517, - "80": 0.11603, - "81": 0.11937, - "82": 0.11678, - "83": 0.11615, - "84": 0.11651, - "85": 0.11591, - "86": 0.11576, - "87": 0.11657, - "88": 0.11645, - "89": 0.11605, - "90": 0.11536, - "91": 0.11542, - "92": 0.11782, - "93": 0.1154, - "94": 0.11637, - "95": 0.11553, - "96": 0.11553, - "97": 0.11617, - "98": 0.11806, - "99": 0.11598, - "100": 0.11537 + "2": 5.26546, + "3": 0.11039, + "4": 1.69732, + "5": 0.69485, + "6": 1.22123, + "7": 2.17274, + "8": 1.65394, + "9": 1.33841, + "10": 0.72951, + "11": 2.726, + "12": 0.30057, + "13": 1.36607, + "14": 0.95454, + "15": 1.20697, + "16": 1.11041, + "17": 0.81611, + "18": 1.63635, + "19": 1.18793, + "20": 0.61355, + "21": 1.23948, + "22": 0.98086, + "23": 1.18426, + "24": 1.44615, + "25": 0.65362, + "26": 1.37892, + "27": 0.58163, + "28": 1.0958, + "29": 1.203, + "30": 0.32791, + "31": 0.40362, + "32": 1.80398, + "33": 0.87381, + "34": 0.59399, + "35": 1.29432, + "36": 1.71768, + "37": 1.32714, + "38": 0.62701, + "39": 1.03792, + "40": 1.38208, + "41": 1.12252, + "42": 0.96824, + "43": 0.81749, + "44": 1.53456, + "45": 0.87838, + "46": 1.15183, + "47": 1.03073, + "48": 0.89897, + "49": 0.84977, + "50": 0.97437, + "51": 0.14347, + "52": 0.14622, + "53": 1.55814, + "54": 1.88392, + "55": 1.206, + "56": 2.15074, + "57": 1.36797, + "58": 1.00075, + "59": 1.60743, + "60": 0.98849, + "61": 0.35381, + "62": 1.08462, + "63": 0.95054, + "64": 1.11392, + "65": 0.92248, + "66": 1.36403, + "67": 2.28419, + "68": 1.10544, + "69": 1.22885, + "70": 1.63402, + "71": 0.35629, + "72": 0.77029, + "73": 1.34906, + "74": 0.59604, + "75": 2.02198, + "76": 0.63959, + "77": 0.79304, + "78": 2.70578, + "79": 0.90997, + "80": 0.81192, + "81": 1.14064, + "82": 1.85665, + "83": 0.19518, + "84": 1.72057, + "85": 0.50786, + "86": 1.97602, + "87": 1.48892, + "88": 1.15139, + "89": 1.51477, + "90": 1.53958, + "91": 0.8016, + "92": 2.25315, + "93": 0.18801, + "94": 1.30516, + "95": 0.92848, + "96": 1.13281, + "97": 1.18005, + "98": 1.29502, + "99": 1.75802, + "100": 1.07422 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json index ecd3d1ee35c..fb3c77f718b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90185, + "1": 10.90187, "2": 10.90476, - "3": 10.91298, - "4": 10.90732, - "5": 10.90338, - "6": 10.88352, - "7": 10.89466, - "8": 10.9003, - "9": 10.90241, - "10": 10.90124, - "11": 10.89365, - "12": 10.88403, - "13": 10.87658, - "14": 10.87083, - "15": 10.86053, - "16": 10.85047, - "17": 10.85366, - "18": 10.83505, - "19": 10.83504, - "20": 10.75556, - "21": 10.75273, - "22": 10.74048, - "23": 10.73225, - "24": 10.6904, - "25": 10.70403, - "26": 10.68252, - "27": 10.64963, - "28": 10.5671, - "29": 10.54339, - "30": 10.51454, - "31": 10.51312, + "3": 10.91307, + "4": 10.90729, + "5": 10.90346, + "6": 10.88351, + "7": 10.89468, + "8": 10.90036, + "9": 10.90242, + "10": 10.90119, + "11": 10.89375, + "12": 10.8841, + "13": 10.8767, + "14": 10.87086, + "15": 10.86051, + "16": 10.85048, + "17": 10.85362, + "18": 10.83516, + "19": 10.83498, + "20": 10.75559, + "21": 10.75274, + "22": 10.74047, + "23": 10.73228, + "24": 10.69032, + "25": 10.70411, + "26": 10.68255, + "27": 10.64974, + "28": 10.56717, + "29": 10.54336, + "30": 10.5146, + "31": 10.51313, "32": 10.49085, - "33": 10.46425, - "34": 10.41243, - "35": 10.41921, - "36": 10.40599, - "37": 10.35423, - "38": 10.36112, - "39": 10.32243, - "40": 10.32058, - "41": 10.28482, - "42": 10.27525, - "43": 10.24427, - "44": 10.21985, + "33": 10.46423, + "34": 10.41249, + "35": 10.41923, + "36": 10.40607, + "37": 10.35428, + "38": 10.36119, + "39": 10.32248, + "40": 10.3206, + "41": 10.28486, + "42": 10.27528, + "43": 10.24433, + "44": 10.21989, "45": 10.23687, - "46": 10.19624, - "47": 10.18389, - "48": 10.1323, - "49": 10.13047, - "50": 10.14436, - "51": 10.14123, - "52": 10.09131, - "53": 10.08918, - "54": 10.06284, - "55": 10.01711, - "56": 10.07149, - "57": 10.03622, - "58": 10.06524, - "59": 10.00246, - "60": 10.01718, - "61": 9.98392, - "62": 9.93675, - "63": 10.03256, - "64": 9.96969, - "65": 9.93232, - "66": 9.97442, - "67": 9.94335, - "68": 9.8968, - "69": 9.9134, - "70": 9.9038, - "71": 9.93369, - "72": 9.89264, - "73": 9.87323, - "74": 9.87541, - "75": 9.84114, - "76": 9.89728, - "77": 9.8872, + "46": 10.19634, + "47": 10.18395, + "48": 10.13233, + "49": 10.13056, + "50": 10.14445, + "51": 10.14128, + "52": 10.09134, + "53": 10.08921, + "54": 10.06288, + "55": 10.01713, + "56": 10.07152, + "57": 10.0363, + "58": 10.06528, + "59": 10.0025, + "60": 10.0172, + "61": 9.98397, + "62": 9.93682, + "63": 10.03263, + "64": 9.9697, + "65": 9.9324, + "66": 9.97451, + "67": 9.94342, + "68": 9.89685, + "69": 9.91349, + "70": 9.90387, + "71": 9.93377, + "72": 9.89268, + "73": 9.87331, + "74": 9.87543, + "75": 9.84116, + "76": 9.89725, + "77": 9.88725, "78": 9.82611, - "79": 9.83323, - "80": 9.8524, - "81": 9.87595, - "82": 9.82806, - "83": 9.7717, - "84": 9.70894, - "85": 9.69359, - "86": 9.80476, - "87": 9.8523, - "88": 9.82792, - "89": 9.74787, - "90": 9.74092, - "91": 9.7535, - "92": 9.74928, - "93": 9.67428, - "94": 9.75078, - "95": 9.75704, - "96": 9.73734, - "97": 9.6548, + "79": 9.83326, + "80": 9.85244, + "81": 9.87596, + "82": 9.82808, + "83": 9.77177, + "84": 9.70902, + "85": 9.69361, + "86": 9.80481, + "87": 9.85232, + "88": 9.82795, + "89": 9.7479, + "90": 9.74098, + "91": 9.75355, + "92": 9.74932, + "93": 9.67432, + "94": 9.7508, + "95": 9.75706, + "96": 9.73746, + "97": 9.65486, "98": 9.69864, - "99": 9.75611, - "100": 9.63811 + "99": 9.75613, + "100": 9.63818 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1985.0, - "2": 1849.0, - "3": 1918.0, - "4": 1845.0, - "5": 1981.0, - "6": 1867.0, - "7": 2134.0, - "8": 1795.0, - "9": 1915.0, - "10": 2000.0, - "11": 1951.0, - "12": 1871.0, - "13": 1991.0, - "14": 2057.0, - "15": 1826.0, - "16": 1941.0, - "17": 2086.0, - "18": 2095.0, - "19": 2002.0, - "20": 1853.0, - "21": 2090.0, - "22": 1944.0, - "23": 1847.0, - "24": 2008.0, - "25": 1831.0, - "26": 2011.0, - "27": 1982.0, - "28": 1928.0, - "29": 2119.0, - "30": 1923.0, - "31": 2072.0, - "32": 1955.0, - "33": 2018.0, - "34": 2088.0, - "35": 2066.0, - "36": 2037.0, - "37": 2227.0, - "38": 2208.0, - "39": 2078.0, - "40": 2359.0, - "41": 2345.0, - "42": 2083.0, - "43": 2417.0, - "44": 2281.0, - "45": 2524.0, - "46": 2404.0, - "47": 2372.0, - "48": 2558.0, - "49": 2718.0, - "50": 2502.0, - "51": 2528.0, - "52": 2618.0, - "53": 2473.0, - "54": 2778.0, - "55": 2518.0, - "56": 2819.0, - "57": 2354.0, - "58": 3243.0, - "59": 2808.0, - "60": 2877.0, - "61": 2627.0, - "62": 2989.0, - "63": 3013.0, - "64": 3270.0, - "65": 2533.0, - "66": 2795.0, - "67": 3328.0, - "68": 3128.0, - "69": 3087.0, - "70": 2901.0, - "71": 3034.0, - "72": 3116.0, - "73": 3270.0, - "74": 3013.0, - "75": 3097.0, - "76": 3258.0, - "77": 3142.0, - "78": 3204.0, + "1": 2015.0, + "2": 1861.0, + "3": 1941.0, + "4": 1935.0, + "5": 1965.0, + "6": 1843.0, + "7": 2203.0, + "8": 1887.0, + "9": 1903.0, + "10": 2052.0, + "11": 2028.0, + "12": 1917.0, + "13": 2102.0, + "14": 2083.0, + "15": 1736.0, + "16": 1963.0, + "17": 2051.0, + "18": 1995.0, + "19": 1887.0, + "20": 1908.0, + "21": 2013.0, + "22": 2004.0, + "23": 1792.0, + "24": 1939.0, + "25": 1931.0, + "26": 2003.0, + "27": 1989.0, + "28": 1899.0, + "29": 2110.0, + "30": 1874.0, + "31": 2037.0, + "32": 1981.0, + "33": 1993.0, + "34": 2156.0, + "35": 2084.0, + "36": 2097.0, + "37": 2228.0, + "38": 2216.0, + "39": 2142.0, + "40": 2334.0, + "41": 2302.0, + "42": 2107.0, + "43": 2369.0, + "44": 2243.0, + "45": 2546.0, + "46": 2378.0, + "47": 2433.0, + "48": 2559.0, + "49": 2660.0, + "50": 2677.0, + "51": 2560.0, + "52": 2678.0, + "53": 2549.0, + "54": 2732.0, + "55": 2532.0, + "56": 2783.0, + "57": 2378.0, + "58": 3179.0, + "59": 2762.0, + "60": 2809.0, + "61": 2644.0, + "62": 2957.0, + "63": 3040.0, + "64": 3145.0, + "65": 2674.0, + "66": 2825.0, + "67": 3388.0, + "68": 3169.0, + "69": 3036.0, + "70": 2791.0, + "71": 2990.0, + "72": 3030.0, + "73": 3211.0, + "74": 3126.0, + "75": 3025.0, + "76": 3267.0, + "77": 3317.0, + "78": 3256.0, "79": 2963.0, - "80": 3126.0, - "81": 3222.0, - "82": 3247.0, - "83": 2893.0, - "84": 2910.0, - "85": 2985.0, - "86": 3218.0, - "87": 3275.0, - "88": 3179.0, - "89": 3082.0, - "90": 3491.0, - "91": 2820.0, - "92": 2940.0, - "93": 2944.0, - "94": 3371.0, - "95": 3239.0, - "96": 3199.0, - "97": 3432.0, - "98": 3457.0, - "99": 3228.0, - "100": 3028.0 + "80": 3088.0, + "81": 3243.0, + "82": 3238.0, + "83": 3035.0, + "84": 2916.0, + "85": 2879.0, + "86": 3213.0, + "87": 3332.0, + "88": 3177.0, + "89": 2975.0, + "90": 3504.0, + "91": 2955.0, + "92": 3004.0, + "93": 3022.0, + "94": 3272.0, + "95": 3238.0, + "96": 3220.0, + "97": 3279.0, + "98": 3385.0, + "99": 3139.0, + "100": 2992.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.52139, - "3": 1.31676, - "4": 0.79975, - "5": 0.76142, - "6": 0.61688, - "7": 1.10994, - "8": 1.21103, - "9": 1.26967, - "10": 0.80736, - "11": 1.03133, - "12": 1.01682, - "13": 1.14165, - "14": 1.17872, - "15": 1.03174, - "16": 0.98198, - "17": 1.20988, - "18": 1.13469, - "19": 0.8773, - "20": 0.87974, - "21": 0.83743, - "22": 1.10107, - "23": 1.1823, - "24": 1.44275, - "25": 0.63836, - "26": 1.00997, - "27": 1.08467, - "28": 1.30864, - "29": 0.74933, - "30": 1.0344, - "31": 0.73649, - "32": 1.78843, - "33": 1.18019, - "34": 0.3539, - "35": 1.39269, - "36": 1.12814, - "37": 0.87369, - "38": 1.17387, - "39": 1.43797, - "40": 1.61001, - "41": 1.11028, - "42": 1.17756, - "43": 0.87467, - "44": 1.52143, - "45": 1.04728, - "46": 0.87365, - "47": 1.2417, - "48": 0.95874, - "49": 0.83764, - "50": 0.97516, - "51": 0.50004, - "52": 1.16182, - "53": 1.048, - "54": 1.25479, - "55": 0.92458, - "56": 0.9998, - "57": 0.87311, - "58": 0.81221, - "59": 0.94875, - "60": 1.04215, - "61": 0.85544, - "62": 1.22979, - "63": 0.7709, - "64": 1.2745, - "65": 0.72474, - "66": 1.08558, - "67": 1.23927, - "68": 0.63327, - "69": 1.35395, - "70": 1.03255, - "71": 0.96629, - "72": 1.06889, - "73": 0.98471, - "74": 0.58585, - "75": 0.88547, - "76": 0.98341, - "77": 0.94806, - "78": 1.08725, - "79": 1.90974, - "80": 1.09005, - "81": 0.91725, - "82": 1.27878, - "83": 1.01817, - "84": 0.80636, - "85": 0.45266, - "86": 0.67137, - "87": 1.50911, - "88": 0.69126, - "89": 0.54835, - "90": 1.20966, - "91": 0.86093, - "92": 1.26522, - "93": 0.90824, - "94": 1.01004, - "95": 0.76243, - "96": 1.13206, - "97": 1.03872, - "98": 1.08888, - "99": 1.4323, - "100": 1.16297 + "2": 3.8564, + "3": 1.1541, + "4": 0.90594, + "5": 0.74436, + "6": 0.82202, + "7": 0.97644, + "8": 1.27802, + "9": 0.95477, + "10": 0.44563, + "11": 0.83763, + "12": 0.85236, + "13": 1.00504, + "14": 0.96244, + "15": 0.67815, + "16": 0.86434, + "17": 0.93862, + "18": 1.01127, + "19": 0.67597, + "20": 0.84048, + "21": 0.8921, + "22": 0.97334, + "23": 0.82081, + "24": 1.00082, + "25": 0.5061, + "26": 0.90623, + "27": 0.75537, + "28": 0.79256, + "29": 0.70379, + "30": 0.59278, + "31": 0.73268, + "32": 1.23419, + "33": 0.91957, + "34": 0.64744, + "35": 1.15605, + "36": 0.68687, + "37": 0.79524, + "38": 0.72546, + "39": 0.98846, + "40": 1.04922, + "41": 1.11489, + "42": 0.83232, + "43": 0.64333, + "44": 0.91465, + "45": 1.00271, + "46": 0.53276, + "47": 1.00047, + "48": 0.912, + "49": 0.90836, + "50": 0.83399, + "51": 0.37169, + "52": 0.88505, + "53": 1.13288, + "54": 1.18632, + "55": 0.90068, + "56": 0.84583, + "57": 0.79393, + "58": 0.87331, + "59": 1.09552, + "60": 0.78349, + "61": 0.92509, + "62": 1.06453, + "63": 0.79956, + "64": 0.93439, + "65": 0.69177, + "66": 0.80772, + "67": 0.87466, + "68": 0.62363, + "69": 1.27191, + "70": 0.75689, + "71": 0.99508, + "72": 0.71588, + "73": 0.86663, + "74": 0.7, + "75": 0.57794, + "76": 1.10323, + "77": 0.94308, + "78": 0.87427, + "79": 1.15388, + "80": 0.67634, + "81": 0.6421, + "82": 0.89638, + "83": 0.93683, + "84": 0.85564, + "85": 0.56603, + "86": 0.86341, + "87": 1.12682, + "88": 0.8837, + "89": 0.89107, + "90": 1.03817, + "91": 0.64312, + "92": 0.96792, + "93": 0.85345, + "94": 0.73455, + "95": 0.85378, + "96": 0.74028, + "97": 0.78249, + "98": 1.00838, + "99": 1.02968, + "100": 0.69008 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json index 43f99881035..e9d05b781aa 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89654, - "2": 10.90691, + "1": 10.89657, + "2": 10.90693, "3": 10.91531, "4": 10.90745, - "5": 10.90126, - "6": 10.88092, - "7": 10.8965, - "8": 10.90076, - "9": 10.90348, - "10": 10.90007, - "11": 10.88795, - "12": 10.88819, - "13": 10.87343, + "5": 10.90137, + "6": 10.88089, + "7": 10.89653, + "8": 10.90074, + "9": 10.9035, + "10": 10.90012, + "11": 10.88799, + "12": 10.88837, + "13": 10.87349, "14": 10.8724, - "15": 10.86247, - "16": 10.84976, + "15": 10.86255, + "16": 10.84972, "17": 10.85501, - "18": 10.83714, - "19": 10.83403, - "20": 10.75411, - "21": 10.75673, - "22": 10.73996, - "23": 10.73089, - "24": 10.68844, - "25": 10.70439, - "26": 10.67871, - "27": 10.64846, - "28": 10.56637, - "29": 10.54447, - "30": 10.51806, - "31": 10.5137, - "32": 10.48935, - "33": 10.46524, - "34": 10.41461, - "35": 10.41939, - "36": 10.40505, - "37": 10.35431, - "38": 10.36341, - "39": 10.32372, + "18": 10.83718, + "19": 10.83399, + "20": 10.75415, + "21": 10.75676, + "22": 10.74004, + "23": 10.73088, + "24": 10.68839, + "25": 10.70444, + "26": 10.67873, + "27": 10.64843, + "28": 10.56643, + "29": 10.54439, + "30": 10.5181, + "31": 10.51374, + "32": 10.48947, + "33": 10.46527, + "34": 10.41467, + "35": 10.4195, + "36": 10.40509, + "37": 10.35434, + "38": 10.36339, + "39": 10.32376, "40": 10.32151, - "41": 10.28356, - "42": 10.27323, - "43": 10.24586, + "41": 10.28361, + "42": 10.27324, + "43": 10.24591, "44": 10.22028, - "45": 10.23662, - "46": 10.19741, - "47": 10.18364, - "48": 10.13263, - "49": 10.13339, - "50": 10.14201, - "51": 10.14228, - "52": 10.09062, - "53": 10.09143, - "54": 10.06527, - "55": 10.01912, - "56": 10.0707, - "57": 10.03678, - "58": 10.06464, - "59": 10.00182, - "60": 10.01777, - "61": 9.98209, - "62": 9.93695, - "63": 10.03326, - "64": 9.97119, - "65": 9.93134, - "66": 9.97363, - "67": 9.94533, - "68": 9.8973, - "69": 9.91404, - "70": 9.90408, - "71": 9.93254, - "72": 9.89151, - "73": 9.87306, - "74": 9.87378, - "75": 9.84018, - "76": 9.89859, - "77": 9.88919, - "78": 9.82547, - "79": 9.83383, - "80": 9.85192, - "81": 9.8766, - "82": 9.82817, - "83": 9.77297, - "84": 9.70944, - "85": 9.69256, - "86": 9.80621, - "87": 9.85442, - "88": 9.82685, - "89": 9.74675, - "90": 9.74116, - "91": 9.75427, - "92": 9.75073, - "93": 9.6732, - "94": 9.75064, + "45": 10.23666, + "46": 10.19742, + "47": 10.18361, + "48": 10.13266, + "49": 10.13354, + "50": 10.14199, + "51": 10.14231, + "52": 10.09069, + "53": 10.09144, + "54": 10.06529, + "55": 10.01922, + "56": 10.07069, + "57": 10.03686, + "58": 10.06469, + "59": 10.00192, + "60": 10.01779, + "61": 9.98216, + "62": 9.937, + "63": 10.03329, + "64": 9.97124, + "65": 9.93143, + "66": 9.97367, + "67": 9.94536, + "68": 9.89742, + "69": 9.9141, + "70": 9.90415, + "71": 9.93259, + "72": 9.89157, + "73": 9.87313, + "74": 9.87381, + "75": 9.84021, + "76": 9.8987, + "77": 9.88927, + "78": 9.82554, + "79": 9.83382, + "80": 9.85193, + "81": 9.87669, + "82": 9.82825, + "83": 9.77307, + "84": 9.70951, + "85": 9.6926, + "86": 9.80619, + "87": 9.85444, + "88": 9.82686, + "89": 9.74679, + "90": 9.7412, + "91": 9.7543, + "92": 9.75074, + "93": 9.67326, + "94": 9.75067, "95": 9.75534, - "96": 9.73655, - "97": 9.65707, - "98": 9.70046, - "99": 9.75769, - "100": 9.63942 + "96": 9.73662, + "97": 9.65713, + "98": 9.7005, + "99": 9.75773, + "100": 9.63948 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2095.0, - "2": 1813.0, - "3": 1964.0, - "4": 1941.0, - "5": 2028.0, - "6": 1802.0, - "7": 2138.0, - "8": 1885.0, - "9": 1926.0, - "10": 1994.0, - "11": 1877.0, - "12": 1919.0, - "13": 2041.0, - "14": 2071.0, - "15": 1764.0, - "16": 1887.0, - "17": 2018.0, - "18": 2021.0, - "19": 1932.0, - "20": 1935.0, - "21": 2124.0, - "22": 1953.0, - "23": 1845.0, - "24": 2083.0, - "25": 1957.0, - "26": 2015.0, - "27": 1922.0, - "28": 1877.0, - "29": 2079.0, - "30": 2018.0, - "31": 2059.0, - "32": 1936.0, - "33": 1952.0, - "34": 2145.0, - "35": 2068.0, - "36": 2130.0, - "37": 2336.0, - "38": 2263.0, - "39": 2200.0, - "40": 2300.0, - "41": 2315.0, - "42": 2050.0, - "43": 2516.0, - "44": 2307.0, - "45": 2648.0, - "46": 2394.0, - "47": 2482.0, - "48": 2524.0, - "49": 2737.0, - "50": 2567.0, - "51": 2694.0, - "52": 2793.0, - "53": 2545.0, - "54": 2757.0, - "55": 2432.0, - "56": 2679.0, - "57": 2320.0, - "58": 3267.0, - "59": 2753.0, - "60": 2858.0, - "61": 2649.0, - "62": 3062.0, - "63": 3062.0, - "64": 3109.0, - "65": 2574.0, - "66": 2883.0, - "67": 3234.0, - "68": 3068.0, - "69": 2999.0, - "70": 2893.0, - "71": 3027.0, - "72": 3014.0, - "73": 3207.0, - "74": 3164.0, - "75": 3074.0, - "76": 3056.0, - "77": 3230.0, - "78": 3138.0, - "79": 2892.0, - "80": 2913.0, - "81": 3320.0, + "1": 2074.0, + "2": 1783.0, + "3": 1986.0, + "4": 1872.0, + "5": 1936.0, + "6": 1853.0, + "7": 2097.0, + "8": 2003.0, + "9": 1931.0, + "10": 1875.0, + "11": 1858.0, + "12": 1915.0, + "13": 2048.0, + "14": 2083.0, + "15": 1774.0, + "16": 1892.0, + "17": 1994.0, + "18": 2064.0, + "19": 1928.0, + "20": 1879.0, + "21": 2025.0, + "22": 1899.0, + "23": 1834.0, + "24": 2006.0, + "25": 1947.0, + "26": 1898.0, + "27": 1918.0, + "28": 1946.0, + "29": 1996.0, + "30": 2010.0, + "31": 2104.0, + "32": 1979.0, + "33": 1957.0, + "34": 2174.0, + "35": 2116.0, + "36": 2149.0, + "37": 2286.0, + "38": 2279.0, + "39": 2187.0, + "40": 2257.0, + "41": 2283.0, + "42": 2079.0, + "43": 2341.0, + "44": 2243.0, + "45": 2579.0, + "46": 2381.0, + "47": 2355.0, + "48": 2521.0, + "49": 2673.0, + "50": 2625.0, + "51": 2617.0, + "52": 2701.0, + "53": 2420.0, + "54": 2742.0, + "55": 2436.0, + "56": 2743.0, + "57": 2367.0, + "58": 3216.0, + "59": 2664.0, + "60": 2875.0, + "61": 2629.0, + "62": 2980.0, + "63": 2945.0, + "64": 3166.0, + "65": 2518.0, + "66": 2841.0, + "67": 3341.0, + "68": 3032.0, + "69": 2995.0, + "70": 2876.0, + "71": 3008.0, + "72": 3073.0, + "73": 3310.0, + "74": 3115.0, + "75": 3076.0, + "76": 3166.0, + "77": 3166.0, + "78": 3159.0, + "79": 2915.0, + "80": 3060.0, + "81": 3241.0, "82": 3212.0, - "83": 2954.0, - "84": 2985.0, - "85": 2808.0, - "86": 3125.0, - "87": 3289.0, - "88": 3166.0, - "89": 3209.0, - "90": 3479.0, - "91": 2751.0, - "92": 3018.0, - "93": 3074.0, - "94": 3362.0, - "95": 3316.0, - "96": 3109.0, - "97": 3352.0, - "98": 3439.0, - "99": 3152.0, - "100": 3173.0 + "83": 3052.0, + "84": 2935.0, + "85": 3009.0, + "86": 3240.0, + "87": 3282.0, + "88": 3164.0, + "89": 3054.0, + "90": 3428.0, + "91": 2696.0, + "92": 3037.0, + "93": 3010.0, + "94": 3393.0, + "95": 3268.0, + "96": 3177.0, + "97": 3262.0, + "98": 3348.0, + "99": 3134.0, + "100": 3061.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 917459968.0, - "2": 917459968.0, - "3": 917459968.0, - "4": 917459968.0, - "5": 917459968.0, - "6": 917459968.0, - "7": 917459968.0, - "8": 917459968.0, - "9": 917459968.0, - "10": 917459968.0, - "11": 917459968.0, - "12": 917459968.0, - "13": 917459968.0, - "14": 917459968.0, - "15": 917459968.0, - "16": 917459968.0, - "17": 917459968.0, - "18": 917459968.0, - "19": 917459968.0, - "20": 917459968.0, - "21": 917459968.0, - "22": 917459968.0, - "23": 917459968.0, - "24": 917459968.0, - "25": 917459968.0, - "26": 917459968.0, - "27": 917459968.0, - "28": 917459968.0, - "29": 917459968.0, - "30": 917459968.0, - "31": 917459968.0, - "32": 917459968.0, - "33": 917459968.0, - "34": 917459968.0, - "35": 917459968.0, - "36": 917459968.0, - "37": 917459968.0, - "38": 917459968.0, - "39": 917459968.0, - "40": 917459968.0, - "41": 917459968.0, - "42": 917459968.0, - "43": 917459968.0, - "44": 917459968.0, - "45": 917459968.0, - "46": 917459968.0, - "47": 917459968.0, - "48": 917459968.0, - "49": 917459968.0, - "50": 917459968.0, - "51": 917459968.0, - "52": 917459968.0, - "53": 917459968.0, - "54": 917459968.0, - "55": 917459968.0, - "56": 917459968.0, - "57": 917459968.0, - "58": 917459968.0, - "59": 917459968.0, - "60": 917459968.0, - "61": 917459968.0, - "62": 917459968.0, - "63": 917459968.0, - "64": 917459968.0, - "65": 917459968.0, - "66": 917459968.0, - "67": 917459968.0, - "68": 917459968.0, - "69": 917459968.0, - "70": 917459968.0, - "71": 917459968.0, - "72": 917459968.0, - "73": 917459968.0, - "74": 917459968.0, - "75": 917459968.0, - "76": 917459968.0, - "77": 917459968.0, - "78": 917459968.0, - "79": 917459968.0, - "80": 917459968.0, - "81": 917459968.0, - "82": 917459968.0, - "83": 917459968.0, - "84": 917459968.0, - "85": 917459968.0, - "86": 917459968.0, - "87": 917459968.0, - "88": 917459968.0, - "89": 917459968.0, - "90": 917459968.0, - "91": 917459968.0, - "92": 917459968.0, - "93": 917459968.0, - "94": 917459968.0, - "95": 917459968.0, - "96": 917459968.0, - "97": 917459968.0, - "98": 917459968.0, - "99": 917459968.0, - "100": 917459968.0 + "1": 923751424.0, + "2": 923751424.0, + "3": 923751424.0, + "4": 923751424.0, + "5": 923751424.0, + "6": 923751424.0, + "7": 923751424.0, + "8": 923751424.0, + "9": 923751424.0, + "10": 923751424.0, + "11": 923751424.0, + "12": 923751424.0, + "13": 923751424.0, + "14": 923751424.0, + "15": 923751424.0, + "16": 923751424.0, + "17": 923751424.0, + "18": 923751424.0, + "19": 923751424.0, + "20": 923751424.0, + "21": 923751424.0, + "22": 923751424.0, + "23": 923751424.0, + "24": 923751424.0, + "25": 923751424.0, + "26": 923751424.0, + "27": 923751424.0, + "28": 923751424.0, + "29": 923751424.0, + "30": 923751424.0, + "31": 923751424.0, + "32": 923751424.0, + "33": 923751424.0, + "34": 923751424.0, + "35": 923751424.0, + "36": 923751424.0, + "37": 923751424.0, + "38": 923751424.0, + "39": 923751424.0, + "40": 923751424.0, + "41": 923751424.0, + "42": 923751424.0, + "43": 923751424.0, + "44": 923751424.0, + "45": 923751424.0, + "46": 923751424.0, + "47": 923751424.0, + "48": 923751424.0, + "49": 923751424.0, + "50": 923751424.0, + "51": 923751424.0, + "52": 923751424.0, + "53": 923751424.0, + "54": 923751424.0, + "55": 923751424.0, + "56": 923751424.0, + "57": 923751424.0, + "58": 923751424.0, + "59": 923751424.0, + "60": 923751424.0, + "61": 923751424.0, + "62": 923751424.0, + "63": 923751424.0, + "64": 923751424.0, + "65": 923751424.0, + "66": 923751424.0, + "67": 923751424.0, + "68": 923751424.0, + "69": 923751424.0, + "70": 923751424.0, + "71": 923751424.0, + "72": 923751424.0, + "73": 923751424.0, + "74": 923751424.0, + "75": 923751424.0, + "76": 923751424.0, + "77": 923751424.0, + "78": 923751424.0, + "79": 923751424.0, + "80": 923751424.0, + "81": 923751424.0, + "82": 923751424.0, + "83": 923751424.0, + "84": 923751424.0, + "85": 923751424.0, + "86": 923751424.0, + "87": 923751424.0, + "88": 923751424.0, + "89": 923751424.0, + "90": 923751424.0, + "91": 923751424.0, + "92": 923751424.0, + "93": 923751424.0, + "94": 923751424.0, + "95": 923751424.0, + "96": 923751424.0, + "97": 923751424.0, + "98": 923751424.0, + "99": 923751424.0, + "100": 923751424.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2266035200.0, - "2": 2624452608.0, - "3": 2624452608.0, - "4": 2624452608.0, - "5": 2624452608.0, - "6": 2624452608.0, - "7": 2624452608.0, - "8": 2624452608.0, - "9": 2624452608.0, - "10": 2624452608.0, - "11": 2624452608.0, - "12": 2624452608.0, - "13": 2624452608.0, - "14": 2624452608.0, - "15": 2624452608.0, - "16": 2624452608.0, - "17": 2624452608.0, - "18": 2624452608.0, - "19": 2624452608.0, - "20": 2624452608.0, - "21": 2624452608.0, - "22": 2624452608.0, - "23": 2624452608.0, - "24": 2624452608.0, - "25": 2624452608.0, - "26": 2624452608.0, - "27": 2624452608.0, - "28": 2624452608.0, - "29": 2624452608.0, - "30": 2624452608.0, - "31": 2624452608.0, - "32": 2624452608.0, - "33": 2624452608.0, - "34": 2624452608.0, - "35": 2624452608.0, - "36": 2624452608.0, - "37": 2624452608.0, - "38": 2624452608.0, - "39": 2624452608.0, - "40": 2624452608.0, - "41": 2624452608.0, - "42": 2624452608.0, - "43": 2624452608.0, - "44": 2624452608.0, - "45": 2624452608.0, - "46": 2624452608.0, - "47": 2624452608.0, - "48": 2624452608.0, - "49": 2624452608.0, - "50": 2624452608.0, - "51": 2624452608.0, - "52": 2624452608.0, - "53": 2624452608.0, - "54": 2624452608.0, - "55": 2624452608.0, - "56": 2624452608.0, - "57": 2624452608.0, - "58": 2624452608.0, - "59": 2624452608.0, - "60": 2624452608.0, - "61": 2624452608.0, - "62": 2624452608.0, - "63": 2624452608.0, - "64": 2624452608.0, - "65": 2624452608.0, - "66": 2624452608.0, - "67": 2624452608.0, - "68": 2624452608.0, - "69": 2624452608.0, - "70": 2624452608.0, - "71": 2624452608.0, - "72": 2624452608.0, - "73": 2624452608.0, - "74": 2624452608.0, - "75": 2624452608.0, - "76": 2624452608.0, - "77": 2624452608.0, - "78": 2624452608.0, - "79": 2624452608.0, - "80": 2624452608.0, - "81": 2624452608.0, - "82": 2624452608.0, - "83": 2624452608.0, - "84": 2624452608.0, - "85": 2624452608.0, - "86": 2624452608.0, - "87": 2624452608.0, - "88": 2624452608.0, - "89": 2624452608.0, - "90": 2624452608.0, - "91": 2624452608.0, - "92": 2624452608.0, - "93": 2624452608.0, - "94": 2624452608.0, - "95": 2624452608.0, - "96": 2624452608.0, - "97": 2624452608.0, - "98": 2624452608.0, - "99": 2624452608.0, - "100": 2624452608.0 + "2": 2630744064.0, + "3": 2630744064.0, + "4": 2630744064.0, + "5": 2630744064.0, + "6": 2630744064.0, + "7": 2630744064.0, + "8": 2630744064.0, + "9": 2630744064.0, + "10": 2630744064.0, + "11": 2630744064.0, + "12": 2630744064.0, + "13": 2630744064.0, + "14": 2630744064.0, + "15": 2630744064.0, + "16": 2630744064.0, + "17": 2630744064.0, + "18": 2630744064.0, + "19": 2630744064.0, + "20": 2630744064.0, + "21": 2630744064.0, + "22": 2630744064.0, + "23": 2630744064.0, + "24": 2630744064.0, + "25": 2630744064.0, + "26": 2630744064.0, + "27": 2630744064.0, + "28": 2630744064.0, + "29": 2630744064.0, + "30": 2630744064.0, + "31": 2630744064.0, + "32": 2630744064.0, + "33": 2630744064.0, + "34": 2630744064.0, + "35": 2630744064.0, + "36": 2630744064.0, + "37": 2630744064.0, + "38": 2630744064.0, + "39": 2630744064.0, + "40": 2630744064.0, + "41": 2630744064.0, + "42": 2630744064.0, + "43": 2630744064.0, + "44": 2630744064.0, + "45": 2630744064.0, + "46": 2630744064.0, + "47": 2630744064.0, + "48": 2630744064.0, + "49": 2630744064.0, + "50": 2630744064.0, + "51": 2630744064.0, + "52": 2630744064.0, + "53": 2630744064.0, + "54": 2630744064.0, + "55": 2630744064.0, + "56": 2630744064.0, + "57": 2630744064.0, + "58": 2630744064.0, + "59": 2630744064.0, + "60": 2630744064.0, + "61": 2630744064.0, + "62": 2630744064.0, + "63": 2630744064.0, + "64": 2630744064.0, + "65": 2630744064.0, + "66": 2630744064.0, + "67": 2630744064.0, + "68": 2630744064.0, + "69": 2630744064.0, + "70": 2630744064.0, + "71": 2630744064.0, + "72": 2630744064.0, + "73": 2630744064.0, + "74": 2630744064.0, + "75": 2630744064.0, + "76": 2630744064.0, + "77": 2630744064.0, + "78": 2630744064.0, + "79": 2630744064.0, + "80": 2630744064.0, + "81": 2630744064.0, + "82": 2630744064.0, + "83": 2630744064.0, + "84": 2630744064.0, + "85": 2630744064.0, + "86": 2630744064.0, + "87": 2630744064.0, + "88": 2630744064.0, + "89": 2630744064.0, + "90": 2630744064.0, + "91": 2630744064.0, + "92": 2630744064.0, + "93": 2630744064.0, + "94": 2630744064.0, + "95": 2630744064.0, + "96": 2630744064.0, + "97": 2630744064.0, + "98": 2630744064.0, + "99": 2630744064.0, + "100": 2630744064.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.44072, - "3": 0.15843, - "4": 0.14366, - "5": 0.14245, - "6": 0.1413, - "7": 0.14198, - "8": 0.14295, - "9": 0.14292, - "10": 0.14076, - "11": 0.14186, - "12": 0.14159, - "13": 0.14242, - "14": 0.14311, - "15": 0.14236, - "16": 0.14273, - "17": 0.14279, - "18": 0.14398, - "19": 0.15396, - "20": 0.1569, - "21": 0.15581, - "22": 0.15986, - "23": 0.17524, - "24": 0.14173, - "25": 0.14117, - "26": 0.14287, - "27": 0.14255, - "28": 0.14283, - "29": 0.14491, - "30": 0.14436, - "31": 0.1413, - "32": 0.14143, - "33": 0.14422, - "34": 0.1414, - "35": 0.14236, - "36": 0.14075, - "37": 0.14212, - "38": 0.14054, - "39": 0.1405, - "40": 0.14238, - "41": 0.14117, - "42": 0.1416, - "43": 0.14195, - "44": 0.14282, - "45": 0.14124, - "46": 0.14039, - "47": 0.1425, - "48": 0.1416, - "49": 0.14141, - "50": 0.1427, - "51": 0.3111, - "52": 0.15431, - "53": 0.17452, - "54": 0.14383, - "55": 0.14205, - "56": 0.14301, - "57": 0.14274, - "58": 0.14415, - "59": 0.14532, - "60": 0.14353, - "61": 0.14354, - "62": 0.14285, - "63": 0.14444, - "64": 0.14375, - "65": 0.14131, - "66": 0.14367, - "67": 0.14371, - "68": 0.14369, - "69": 0.1439, - "70": 0.14375, - "71": 0.14467, - "72": 0.14324, - "73": 0.14295, - "74": 0.14396, - "75": 0.14589, - "76": 0.14315, - "77": 0.14317, - "78": 0.14103, - "79": 0.14178, - "80": 0.14367, - "81": 0.14247, - "82": 0.14116, - "83": 0.1408, - "84": 0.14131, - "85": 0.14164, - "86": 0.1414, - "87": 0.14325, - "88": 0.14303, - "89": 0.14258, - "90": 0.14187, - "91": 0.1422, - "92": 0.14226, - "93": 0.14088, - "94": 0.14219, - "95": 0.14282, - "96": 0.14366, - "97": 0.14177, - "98": 0.14352, - "99": 0.14279, - "100": 0.14266 + "2": 6.94149, + "3": 0.20062, + "4": 0.20048, + "5": 0.21127, + "6": 0.19541, + "7": 0.18128, + "8": 0.11835, + "9": 0.11967, + "10": 0.1177, + "11": 0.11891, + "12": 0.11973, + "13": 0.12118, + "14": 0.12057, + "15": 0.11991, + "16": 0.11996, + "17": 0.12053, + "18": 0.12069, + "19": 0.11945, + "20": 0.11835, + "21": 0.12044, + "22": 0.12407, + "23": 0.12303, + "24": 0.1211, + "25": 0.12021, + "26": 0.12133, + "27": 0.12267, + "28": 0.12135, + "29": 0.11912, + "30": 0.121, + "31": 0.11947, + "32": 0.11998, + "33": 0.12002, + "34": 0.11904, + "35": 0.11927, + "36": 0.11922, + "37": 0.11965, + "38": 0.12361, + "39": 0.12191, + "40": 0.11944, + "41": 0.11919, + "42": 0.11971, + "43": 0.12013, + "44": 0.12163, + "45": 0.11965, + "46": 0.12049, + "47": 0.12356, + "48": 0.11906, + "49": 0.11982, + "50": 0.11961, + "51": 0.31418, + "52": 0.16013, + "53": 0.12312, + "54": 0.12061, + "55": 0.12181, + "56": 0.11829, + "57": 0.11754, + "58": 0.12315, + "59": 0.12038, + "60": 0.12063, + "61": 0.11654, + "62": 0.1193, + "63": 0.1182, + "64": 0.11951, + "65": 0.12375, + "66": 0.12133, + "67": 0.12058, + "68": 0.11873, + "69": 0.11837, + "70": 0.12108, + "71": 0.11937, + "72": 0.12028, + "73": 0.12002, + "74": 0.12008, + "75": 0.12245, + "76": 0.12235, + "77": 0.12302, + "78": 0.12006, + "79": 0.11937, + "80": 0.1212, + "81": 0.12172, + "82": 0.12214, + "83": 0.12192, + "84": 0.12046, + "85": 0.12147, + "86": 0.11743, + "87": 0.12027, + "88": 0.11952, + "89": 0.11953, + "90": 0.11933, + "91": 0.12132, + "92": 0.12304, + "93": 0.11803, + "94": 0.12188, + "95": 0.12174, + "96": 0.12336, + "97": 0.11915, + "98": 0.12296, + "99": 0.12232, + "100": 0.12113 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json index 9ae0664b7b0..8d16072d30f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96078, - "2": 10.94368, - "3": 10.93755, - "4": 10.95045, - "5": 10.94855, - "6": 10.94271, - "7": 10.93509, - "8": 10.94175, - "9": 10.9418, - "10": 10.92772, - "11": 10.93415, - "12": 10.94481, - "13": 10.92208, - "14": 10.92272, - "15": 10.89966, + "1": 10.96081, + "2": 10.94369, + "3": 10.93753, + "4": 10.9505, + "5": 10.94843, + "6": 10.94269, + "7": 10.93505, + "8": 10.9418, + "9": 10.94177, + "10": 10.9278, + "11": 10.93407, + "12": 10.94483, + "13": 10.9221, + "14": 10.92271, + "15": 10.89965, "16": 10.89041, - "17": 10.88898, - "18": 10.88212, - "19": 10.87681, - "20": 10.80091, - "21": 10.7814, - "22": 10.76174, - "23": 10.76213, - "24": 10.73664, - "25": 10.73447, - "26": 10.72101, - "27": 10.68102, - "28": 10.59583, - "29": 10.57443, - "30": 10.55201, - "31": 10.55316, - "32": 10.52965, + "17": 10.88902, + "18": 10.88214, + "19": 10.87673, + "20": 10.801, + "21": 10.78143, + "22": 10.76172, + "23": 10.76224, + "24": 10.73665, + "25": 10.73451, + "26": 10.72103, + "27": 10.68111, + "28": 10.59594, + "29": 10.57444, + "30": 10.55211, + "31": 10.55324, + "32": 10.52967, "33": 10.49304, - "34": 10.46592, - "35": 10.46781, - "36": 10.43445, - "37": 10.40557, - "38": 10.40698, - "39": 10.38092, - "40": 10.35372, - "41": 10.33113, - "42": 10.31728, - "43": 10.27747, - "44": 10.27475, - "45": 10.27088, - "46": 10.2394, - "47": 10.22002, - "48": 10.17409, - "49": 10.18283, - "50": 10.18254, - "51": 10.17736, - "52": 10.12545, - "53": 10.11938, - "54": 10.09849, - "55": 10.06505, - "56": 10.09606, - "57": 10.0703, - "58": 10.08109, - "59": 10.02394, - "60": 10.04465, - "61": 10.00552, - "62": 9.96193, - "63": 10.0445, - "64": 9.98729, - "65": 9.94156, - "66": 9.99581, - "67": 9.96348, - "68": 9.92327, - "69": 9.92556, - "70": 9.91456, - "71": 9.94802, - "72": 9.91519, - "73": 9.89109, - "74": 9.88876, - "75": 9.84728, - "76": 9.896, - "77": 9.89059, - "78": 9.84155, - "79": 9.83898, - "80": 9.85635, - "81": 9.88226, - "82": 9.83206, - "83": 9.77654, - "84": 9.71211, - "85": 9.69857, - "86": 9.81127, - "87": 9.84704, - "88": 9.81952, - "89": 9.74322, - "90": 9.74316, - "91": 9.75539, - "92": 9.75092, - "93": 9.66352, - "94": 9.74704, - "95": 9.74356, - "96": 9.72114, - "97": 9.65242, - "98": 9.68127, - "99": 9.75572, - "100": 9.62796 + "34": 10.46601, + "35": 10.46785, + "36": 10.43451, + "37": 10.40567, + "38": 10.40699, + "39": 10.38093, + "40": 10.35377, + "41": 10.33117, + "42": 10.31723, + "43": 10.27752, + "44": 10.27479, + "45": 10.27097, + "46": 10.23942, + "47": 10.2201, + "48": 10.17413, + "49": 10.18288, + "50": 10.18261, + "51": 10.17743, + "52": 10.1255, + "53": 10.1194, + "54": 10.09854, + "55": 10.06517, + "56": 10.09613, + "57": 10.07035, + "58": 10.08113, + "59": 10.02402, + "60": 10.04469, + "61": 10.00553, + "62": 9.96201, + "63": 10.04454, + "64": 9.98737, + "65": 9.94162, + "66": 9.99585, + "67": 9.96358, + "68": 9.92335, + "69": 9.9256, + "70": 9.91461, + "71": 9.94806, + "72": 9.91522, + "73": 9.89107, + "74": 9.88884, + "75": 9.84737, + "76": 9.89606, + "77": 9.89063, + "78": 9.84159, + "79": 9.83907, + "80": 9.85636, + "81": 9.8823, + "82": 9.83207, + "83": 9.77662, + "84": 9.71215, + "85": 9.69867, + "86": 9.81131, + "87": 9.84706, + "88": 9.81954, + "89": 9.74321, + "90": 9.74319, + "91": 9.75545, + "92": 9.751, + "93": 9.6636, + "94": 9.74709, + "95": 9.74364, + "96": 9.72117, + "97": 9.65251, + "98": 9.68136, + "99": 9.75576, + "100": 9.62804 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1951.0, - "2": 1836.0, - "3": 1829.0, - "4": 1977.0, - "5": 1855.0, - "6": 1949.0, - "7": 2113.0, - "8": 1918.0, - "9": 1917.0, - "10": 1879.0, - "11": 1898.0, - "12": 1857.0, - "13": 1993.0, - "14": 1891.0, - "15": 1785.0, - "16": 1893.0, - "17": 1913.0, - "18": 1905.0, - "19": 1905.0, - "20": 1878.0, - "21": 1993.0, - "22": 1888.0, - "23": 1913.0, - "24": 2001.0, - "25": 1886.0, - "26": 1952.0, - "27": 2053.0, - "28": 1923.0, - "29": 1995.0, - "30": 1986.0, - "31": 2097.0, - "32": 1946.0, - "33": 1942.0, - "34": 2090.0, - "35": 2120.0, - "36": 2074.0, - "37": 2211.0, - "38": 2238.0, - "39": 2252.0, - "40": 2414.0, - "41": 2320.0, - "42": 2143.0, - "43": 2265.0, - "44": 2227.0, - "45": 2362.0, - "46": 2460.0, - "47": 2462.0, - "48": 2501.0, - "49": 2790.0, - "50": 2607.0, - "51": 2561.0, - "52": 2482.0, - "53": 2547.0, - "54": 2803.0, - "55": 2441.0, - "56": 2657.0, - "57": 2351.0, - "58": 3218.0, - "59": 2723.0, - "60": 2830.0, - "61": 2533.0, - "62": 2975.0, - "63": 3074.0, - "64": 3077.0, - "65": 2643.0, - "66": 2867.0, - "67": 3265.0, - "68": 2940.0, - "69": 2946.0, - "70": 2812.0, - "71": 2922.0, - "72": 3002.0, - "73": 3131.0, - "74": 3011.0, - "75": 2877.0, - "76": 3099.0, - "77": 3165.0, - "78": 3083.0, - "79": 2875.0, - "80": 2910.0, - "81": 3101.0, - "82": 3122.0, - "83": 2955.0, - "84": 2874.0, - "85": 2745.0, - "86": 3208.0, - "87": 3043.0, - "88": 3223.0, - "89": 3100.0, - "90": 3461.0, - "91": 2883.0, - "92": 3192.0, - "93": 2895.0, - "94": 3337.0, - "95": 3129.0, - "96": 3285.0, - "97": 3254.0, - "98": 3248.0, - "99": 2989.0, - "100": 3047.0 + "1": 1992.0, + "2": 1841.0, + "3": 1914.0, + "4": 1948.0, + "5": 1885.0, + "6": 1882.0, + "7": 2104.0, + "8": 1794.0, + "9": 2000.0, + "10": 1885.0, + "11": 1969.0, + "12": 1822.0, + "13": 1976.0, + "14": 1912.0, + "15": 1777.0, + "16": 1918.0, + "17": 1910.0, + "18": 1975.0, + "19": 1918.0, + "20": 1814.0, + "21": 2015.0, + "22": 1917.0, + "23": 1851.0, + "24": 1993.0, + "25": 1942.0, + "26": 1890.0, + "27": 2014.0, + "28": 1874.0, + "29": 1920.0, + "30": 1933.0, + "31": 2018.0, + "32": 2008.0, + "33": 1946.0, + "34": 2065.0, + "35": 2135.0, + "36": 2072.0, + "37": 2212.0, + "38": 2241.0, + "39": 2200.0, + "40": 2444.0, + "41": 2309.0, + "42": 2099.0, + "43": 2373.0, + "44": 2276.0, + "45": 2524.0, + "46": 2509.0, + "47": 2404.0, + "48": 2468.0, + "49": 2814.0, + "50": 2574.0, + "51": 2497.0, + "52": 2399.0, + "53": 2556.0, + "54": 2739.0, + "55": 2386.0, + "56": 2640.0, + "57": 2399.0, + "58": 3307.0, + "59": 2740.0, + "60": 2792.0, + "61": 2740.0, + "62": 2861.0, + "63": 3123.0, + "64": 3132.0, + "65": 2624.0, + "66": 2892.0, + "67": 3351.0, + "68": 3085.0, + "69": 2930.0, + "70": 2914.0, + "71": 2933.0, + "72": 3119.0, + "73": 3123.0, + "74": 3019.0, + "75": 2916.0, + "76": 3189.0, + "77": 3192.0, + "78": 2984.0, + "79": 2871.0, + "80": 2980.0, + "81": 3099.0, + "82": 3032.0, + "83": 2914.0, + "84": 3029.0, + "85": 2896.0, + "86": 3192.0, + "87": 2988.0, + "88": 3174.0, + "89": 3025.0, + "90": 3406.0, + "91": 2881.0, + "92": 3026.0, + "93": 2989.0, + "94": 3342.0, + "95": 3193.0, + "96": 3319.0, + "97": 3268.0, + "98": 3211.0, + "99": 3120.0, + "100": 2925.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 919557120.0, - "2": 919557120.0, - "3": 919557120.0, - "4": 919557120.0, - "5": 919557120.0, - "6": 919557120.0, - "7": 919557120.0, - "8": 919557120.0, - "9": 919557120.0, - "10": 919557120.0, - "11": 919557120.0, - "12": 919557120.0, - "13": 919557120.0, - "14": 919557120.0, - "15": 919557120.0, - "16": 919557120.0, - "17": 919557120.0, - "18": 919557120.0, - "19": 919557120.0, - "20": 919557120.0, - "21": 919557120.0, - "22": 919557120.0, - "23": 919557120.0, - "24": 919557120.0, - "25": 919557120.0, - "26": 919557120.0, - "27": 919557120.0, - "28": 919557120.0, - "29": 919557120.0, - "30": 919557120.0, - "31": 919557120.0, - "32": 919557120.0, - "33": 919557120.0, - "34": 919557120.0, - "35": 919557120.0, - "36": 919557120.0, - "37": 919557120.0, - "38": 919557120.0, - "39": 919557120.0, - "40": 919557120.0, - "41": 919557120.0, - "42": 919557120.0, - "43": 919557120.0, - "44": 919557120.0, - "45": 919557120.0, - "46": 919557120.0, - "47": 919557120.0, - "48": 919557120.0, - "49": 919557120.0, - "50": 919557120.0, - "51": 919557120.0, - "52": 919557120.0, - "53": 919557120.0, - "54": 919557120.0, - "55": 919557120.0, - "56": 919557120.0, - "57": 919557120.0, - "58": 919557120.0, - "59": 919557120.0, - "60": 919557120.0, - "61": 919557120.0, - "62": 919557120.0, - "63": 919557120.0, - "64": 919557120.0, - "65": 919557120.0, - "66": 919557120.0, - "67": 919557120.0, - "68": 919557120.0, - "69": 919557120.0, - "70": 919557120.0, - "71": 919557120.0, - "72": 919557120.0, - "73": 919557120.0, - "74": 919557120.0, - "75": 919557120.0, - "76": 919557120.0, - "77": 919557120.0, - "78": 919557120.0, - "79": 919557120.0, - "80": 919557120.0, - "81": 919557120.0, - "82": 919557120.0, - "83": 919557120.0, - "84": 919557120.0, - "85": 919557120.0, - "86": 919557120.0, - "87": 919557120.0, - "88": 919557120.0, - "89": 919557120.0, - "90": 919557120.0, - "91": 919557120.0, - "92": 919557120.0, - "93": 919557120.0, - "94": 919557120.0, - "95": 919557120.0, - "96": 919557120.0, - "97": 919557120.0, - "98": 919557120.0, - "99": 919557120.0, - "100": 919557120.0 + "1": 917459968.0, + "2": 917459968.0, + "3": 917459968.0, + "4": 917459968.0, + "5": 917459968.0, + "6": 917459968.0, + "7": 917459968.0, + "8": 917459968.0, + "9": 917459968.0, + "10": 917459968.0, + "11": 917459968.0, + "12": 917459968.0, + "13": 917459968.0, + "14": 917459968.0, + "15": 917459968.0, + "16": 917459968.0, + "17": 917459968.0, + "18": 917459968.0, + "19": 917459968.0, + "20": 917459968.0, + "21": 917459968.0, + "22": 917459968.0, + "23": 917459968.0, + "24": 917459968.0, + "25": 917459968.0, + "26": 917459968.0, + "27": 917459968.0, + "28": 917459968.0, + "29": 917459968.0, + "30": 917459968.0, + "31": 917459968.0, + "32": 917459968.0, + "33": 917459968.0, + "34": 917459968.0, + "35": 917459968.0, + "36": 917459968.0, + "37": 917459968.0, + "38": 917459968.0, + "39": 917459968.0, + "40": 917459968.0, + "41": 917459968.0, + "42": 917459968.0, + "43": 917459968.0, + "44": 917459968.0, + "45": 917459968.0, + "46": 917459968.0, + "47": 917459968.0, + "48": 917459968.0, + "49": 917459968.0, + "50": 917459968.0, + "51": 917459968.0, + "52": 917459968.0, + "53": 917459968.0, + "54": 917459968.0, + "55": 917459968.0, + "56": 917459968.0, + "57": 917459968.0, + "58": 917459968.0, + "59": 917459968.0, + "60": 917459968.0, + "61": 917459968.0, + "62": 917459968.0, + "63": 917459968.0, + "64": 917459968.0, + "65": 917459968.0, + "66": 917459968.0, + "67": 917459968.0, + "68": 917459968.0, + "69": 917459968.0, + "70": 917459968.0, + "71": 917459968.0, + "72": 917459968.0, + "73": 917459968.0, + "74": 917459968.0, + "75": 917459968.0, + "76": 917459968.0, + "77": 917459968.0, + "78": 917459968.0, + "79": 917459968.0, + "80": 917459968.0, + "81": 917459968.0, + "82": 917459968.0, + "83": 917459968.0, + "84": 917459968.0, + "85": 917459968.0, + "86": 917459968.0, + "87": 917459968.0, + "88": 917459968.0, + "89": 917459968.0, + "90": 917459968.0, + "91": 917459968.0, + "92": 917459968.0, + "93": 917459968.0, + "94": 917459968.0, + "95": 917459968.0, + "96": 917459968.0, + "97": 917459968.0, + "98": 917459968.0, + "99": 917459968.0, + "100": 917459968.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2240869376.0, - "2": 2601383936.0, - "3": 2601383936.0, - "4": 2601383936.0, - "5": 2601383936.0, - "6": 2601383936.0, - "7": 2601383936.0, - "8": 2601383936.0, - "9": 2601383936.0, - "10": 2601383936.0, - "11": 2601383936.0, - "12": 2601383936.0, - "13": 2601383936.0, - "14": 2601383936.0, - "15": 2601383936.0, - "16": 2601383936.0, - "17": 2601383936.0, - "18": 2601383936.0, - "19": 2601383936.0, - "20": 2601383936.0, - "21": 2601383936.0, - "22": 2601383936.0, - "23": 2601383936.0, - "24": 2601383936.0, - "25": 2601383936.0, - "26": 2601383936.0, - "27": 2601383936.0, - "28": 2601383936.0, - "29": 2601383936.0, - "30": 2601383936.0, - "31": 2601383936.0, - "32": 2601383936.0, - "33": 2601383936.0, - "34": 2601383936.0, - "35": 2601383936.0, - "36": 2601383936.0, - "37": 2601383936.0, - "38": 2601383936.0, - "39": 2601383936.0, - "40": 2601383936.0, - "41": 2601383936.0, - "42": 2601383936.0, - "43": 2601383936.0, - "44": 2601383936.0, - "45": 2601383936.0, - "46": 2601383936.0, - "47": 2601383936.0, - "48": 2601383936.0, - "49": 2601383936.0, - "50": 2601383936.0, - "51": 2601383936.0, - "52": 2601383936.0, - "53": 2601383936.0, - "54": 2601383936.0, - "55": 2601383936.0, - "56": 2601383936.0, - "57": 2601383936.0, - "58": 2601383936.0, - "59": 2601383936.0, - "60": 2601383936.0, - "61": 2601383936.0, - "62": 2601383936.0, - "63": 2601383936.0, - "64": 2601383936.0, - "65": 2601383936.0, - "66": 2601383936.0, - "67": 2601383936.0, - "68": 2601383936.0, - "69": 2601383936.0, - "70": 2601383936.0, - "71": 2601383936.0, - "72": 2601383936.0, - "73": 2601383936.0, - "74": 2601383936.0, - "75": 2601383936.0, - "76": 2601383936.0, - "77": 2601383936.0, - "78": 2601383936.0, - "79": 2601383936.0, - "80": 2601383936.0, - "81": 2601383936.0, - "82": 2601383936.0, - "83": 2601383936.0, - "84": 2601383936.0, - "85": 2601383936.0, - "86": 2601383936.0, - "87": 2601383936.0, - "88": 2601383936.0, - "89": 2601383936.0, - "90": 2601383936.0, - "91": 2601383936.0, - "92": 2601383936.0, - "93": 2601383936.0, - "94": 2601383936.0, - "95": 2601383936.0, - "96": 2601383936.0, - "97": 2601383936.0, - "98": 2601383936.0, - "99": 2601383936.0, - "100": 2601383936.0 + "2": 2599286784.0, + "3": 2599286784.0, + "4": 2599286784.0, + "5": 2599286784.0, + "6": 2599286784.0, + "7": 2599286784.0, + "8": 2599286784.0, + "9": 2599286784.0, + "10": 2599286784.0, + "11": 2599286784.0, + "12": 2599286784.0, + "13": 2599286784.0, + "14": 2599286784.0, + "15": 2599286784.0, + "16": 2599286784.0, + "17": 2599286784.0, + "18": 2599286784.0, + "19": 2599286784.0, + "20": 2599286784.0, + "21": 2599286784.0, + "22": 2599286784.0, + "23": 2599286784.0, + "24": 2599286784.0, + "25": 2599286784.0, + "26": 2599286784.0, + "27": 2599286784.0, + "28": 2599286784.0, + "29": 2599286784.0, + "30": 2599286784.0, + "31": 2599286784.0, + "32": 2599286784.0, + "33": 2599286784.0, + "34": 2599286784.0, + "35": 2599286784.0, + "36": 2599286784.0, + "37": 2599286784.0, + "38": 2599286784.0, + "39": 2599286784.0, + "40": 2599286784.0, + "41": 2599286784.0, + "42": 2599286784.0, + "43": 2599286784.0, + "44": 2599286784.0, + "45": 2599286784.0, + "46": 2599286784.0, + "47": 2599286784.0, + "48": 2599286784.0, + "49": 2599286784.0, + "50": 2599286784.0, + "51": 2599286784.0, + "52": 2599286784.0, + "53": 2599286784.0, + "54": 2599286784.0, + "55": 2599286784.0, + "56": 2599286784.0, + "57": 2599286784.0, + "58": 2599286784.0, + "59": 2599286784.0, + "60": 2599286784.0, + "61": 2599286784.0, + "62": 2599286784.0, + "63": 2599286784.0, + "64": 2599286784.0, + "65": 2599286784.0, + "66": 2599286784.0, + "67": 2599286784.0, + "68": 2599286784.0, + "69": 2599286784.0, + "70": 2599286784.0, + "71": 2599286784.0, + "72": 2599286784.0, + "73": 2599286784.0, + "74": 2599286784.0, + "75": 2599286784.0, + "76": 2599286784.0, + "77": 2599286784.0, + "78": 2599286784.0, + "79": 2599286784.0, + "80": 2599286784.0, + "81": 2599286784.0, + "82": 2599286784.0, + "83": 2599286784.0, + "84": 2599286784.0, + "85": 2599286784.0, + "86": 2599286784.0, + "87": 2599286784.0, + "88": 2599286784.0, + "89": 2599286784.0, + "90": 2599286784.0, + "91": 2599286784.0, + "92": 2599286784.0, + "93": 2599286784.0, + "94": 2599286784.0, + "95": 2599286784.0, + "96": 2599286784.0, + "97": 2599286784.0, + "98": 2599286784.0, + "99": 2599286784.0, + "100": 2599286784.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.79089, - "3": 0.12042, - "4": 1.33387, - "5": 0.42202, - "6": 0.74472, - "7": 1.03792, - "8": 1.10582, - "9": 0.69905, - "10": 0.35146, - "11": 0.98439, - "12": 0.75226, - "13": 0.47222, - "14": 0.81203, - "15": 0.58281, - "16": 0.59479, - "17": 0.58081, - "18": 1.02468, - "19": 0.32551, - "20": 0.67874, - "21": 0.74181, - "22": 0.68122, - "23": 0.60888, - "24": 0.78068, - "25": 0.11827, - "26": 1.28153, - "27": 0.27623, - "28": 0.4275, - "29": 1.02066, - "30": 0.24745, - "31": 0.19409, - "32": 0.85703, - "33": 0.71863, - "34": 0.72464, - "35": 0.68259, - "36": 0.53577, - "37": 0.69604, - "38": 0.48495, - "39": 0.68369, - "40": 0.73251, - "41": 1.04964, - "42": 0.39034, - "43": 0.29345, - "44": 1.01337, - "45": 0.51695, - "46": 0.58453, - "47": 0.83784, - "48": 0.59328, - "49": 0.27611, - "50": 0.93425, - "51": 0.15286, - "52": 0.15528, - "53": 0.78202, - "54": 0.79178, - "55": 0.64843, - "56": 0.39849, - "57": 0.62845, - "58": 0.5124, - "59": 0.59444, - "60": 0.61188, - "61": 0.17017, - "62": 0.87485, - "63": 0.13551, - "64": 0.94336, - "65": 0.63567, - "66": 0.67407, - "67": 0.6575, - "68": 0.22685, - "69": 0.63034, - "70": 0.87896, - "71": 0.2711, - "72": 0.54691, - "73": 0.68699, - "74": 0.41229, - "75": 0.83286, - "76": 0.38755, - "77": 0.49268, - "78": 1.4279, - "79": 0.22921, - "80": 0.43892, - "81": 0.47558, - "82": 0.75499, - "83": 0.36493, - "84": 0.59091, - "85": 0.2452, - "86": 0.49955, - "87": 1.08337, - "88": 0.30985, - "89": 0.73216, - "90": 0.38487, - "91": 0.39271, - "92": 0.70033, - "93": 0.26728, - "94": 0.54077, - "95": 0.51017, - "96": 1.31935, - "97": 0.38261, - "98": 0.43974, - "99": 0.78558, - "100": 0.46287 + "2": 4.43846, + "3": 0.13522, + "4": 0.13248, + "5": 0.13009, + "6": 0.13002, + "7": 0.13586, + "8": 0.13566, + "9": 0.13237, + "10": 0.13265, + "11": 0.13377, + "12": 0.13267, + "13": 0.12999, + "14": 0.13349, + "15": 0.13148, + "16": 0.13432, + "17": 0.13266, + "18": 0.13159, + "19": 0.13306, + "20": 0.13389, + "21": 0.13193, + "22": 0.13277, + "23": 0.13331, + "24": 0.13187, + "25": 0.13278, + "26": 0.13196, + "27": 0.13412, + "28": 0.13171, + "29": 0.1318, + "30": 0.13193, + "31": 0.1313, + "32": 0.1338, + "33": 0.13119, + "34": 0.13324, + "35": 0.13459, + "36": 0.13118, + "37": 0.13328, + "38": 0.13395, + "39": 0.13223, + "40": 0.13148, + "41": 0.13499, + "42": 0.13518, + "43": 0.13343, + "44": 0.13506, + "45": 0.13661, + "46": 0.13329, + "47": 0.13439, + "48": 0.13272, + "49": 0.13309, + "50": 0.13352, + "51": 0.16126, + "52": 0.15722, + "53": 0.1354, + "54": 0.132, + "55": 0.1325, + "56": 0.13166, + "57": 0.13568, + "58": 0.13149, + "59": 0.13106, + "60": 0.13187, + "61": 0.1329, + "62": 0.13178, + "63": 0.13145, + "64": 0.13015, + "65": 0.13228, + "66": 0.13321, + "67": 0.13271, + "68": 0.13349, + "69": 0.13224, + "70": 0.13285, + "71": 0.1322, + "72": 0.13327, + "73": 0.1361, + "74": 0.13329, + "75": 0.13165, + "76": 0.13218, + "77": 0.13174, + "78": 0.13388, + "79": 0.13473, + "80": 0.13272, + "81": 0.13304, + "82": 0.1327, + "83": 0.133, + "84": 0.13255, + "85": 0.1325, + "86": 0.13202, + "87": 0.13198, + "88": 0.13206, + "89": 0.13275, + "90": 0.13235, + "91": 0.13708, + "92": 0.13365, + "93": 0.13353, + "94": 0.13338, + "95": 0.13382, + "96": 0.13263, + "97": 0.13236, + "98": 0.13234, + "99": 0.13405, + "100": 0.1335 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json index 2840b9746ab..b2b82197736 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90174, - "2": 10.90447, - "3": 10.91297, - "4": 10.90733, - "5": 10.90344, - "6": 10.88332, - "7": 10.89468, - "8": 10.90028, - "9": 10.9021, - "10": 10.90121, - "11": 10.89368, - "12": 10.88437, - "13": 10.87653, - "14": 10.87083, - "15": 10.86039, - "16": 10.85074, - "17": 10.85348, - "18": 10.83498, - "19": 10.83514, - "20": 10.75544, - "21": 10.7526, + "1": 10.90178, + "2": 10.90453, + "3": 10.91307, + "4": 10.9073, + "5": 10.90351, + "6": 10.8833, + "7": 10.89473, + "8": 10.90035, + "9": 10.90203, + "10": 10.90113, + "11": 10.8938, + "12": 10.88447, + "13": 10.87658, + "14": 10.87087, + "15": 10.86037, + "16": 10.85073, + "17": 10.85346, + "18": 10.83506, + "19": 10.83511, + "20": 10.75546, + "21": 10.75262, "22": 10.74059, - "23": 10.73225, - "24": 10.69032, - "25": 10.70418, - "26": 10.6823, - "27": 10.64975, - "28": 10.56722, + "23": 10.73224, + "24": 10.69026, + "25": 10.70424, + "26": 10.68233, + "27": 10.64986, + "28": 10.56727, "29": 10.54333, - "30": 10.51478, - "31": 10.5135, - "32": 10.49083, + "30": 10.5149, + "31": 10.51349, + "32": 10.49085, "33": 10.46441, - "34": 10.41257, - "35": 10.41932, - "36": 10.40629, - "37": 10.35435, - "38": 10.36121, - "39": 10.32238, - "40": 10.3208, - "41": 10.28486, - "42": 10.2753, - "43": 10.24448, - "44": 10.22008, + "34": 10.41259, + "35": 10.41936, + "36": 10.40637, + "37": 10.35437, + "38": 10.36126, + "39": 10.32246, + "40": 10.32086, + "41": 10.28493, + "42": 10.2754, + "43": 10.24451, + "44": 10.22011, "45": 10.23701, - "46": 10.19652, - "47": 10.184, + "46": 10.19654, + "47": 10.18403, "48": 10.13231, - "49": 10.13082, - "50": 10.14454, - "51": 10.14126, - "52": 10.09149, - "53": 10.08937, - "54": 10.06294, - "55": 10.01717, - "56": 10.07169, - "57": 10.03655, - "58": 10.06554, - "59": 10.00273, - "60": 10.01727, - "61": 9.98398, - "62": 9.937, - "63": 10.03276, - "64": 9.96992, - "65": 9.93251, - "66": 9.97456, - "67": 9.94352, - "68": 9.89683, - "69": 9.91356, - "70": 9.904, - "71": 9.93385, - "72": 9.8928, - "73": 9.87347, - "74": 9.87565, - "75": 9.84138, - "76": 9.89744, - "77": 9.88738, - "78": 9.82628, - "79": 9.83349, - "80": 9.85242, + "49": 10.13088, + "50": 10.14464, + "51": 10.14125, + "52": 10.09151, + "53": 10.08944, + "54": 10.063, + "55": 10.01719, + "56": 10.07167, + "57": 10.03664, + "58": 10.06557, + "59": 10.00279, + "60": 10.01724, + "61": 9.98395, + "62": 9.93711, + "63": 10.03279, + "64": 9.96997, + "65": 9.93255, + "66": 9.97462, + "67": 9.94357, + "68": 9.89688, + "69": 9.91361, + "70": 9.90406, + "71": 9.93394, + "72": 9.89285, + "73": 9.8735, + "74": 9.87572, + "75": 9.84139, + "76": 9.89746, + "77": 9.88736, + "78": 9.82633, + "79": 9.8335, + "80": 9.85244, "81": 9.876, - "82": 9.82812, - "83": 9.77197, - "84": 9.70915, - "85": 9.6939, - "86": 9.80488, - "87": 9.85235, - "88": 9.82811, - "89": 9.74806, - "90": 9.74109, - "91": 9.75365, - "92": 9.74941, - "93": 9.67469, + "82": 9.82819, + "83": 9.772, + "84": 9.70926, + "85": 9.694, + "86": 9.8049, + "87": 9.85239, + "88": 9.82812, + "89": 9.74807, + "90": 9.7412, + "91": 9.75372, + "92": 9.74942, + "93": 9.67471, "94": 9.75091, - "95": 9.75728, - "96": 9.73751, - "97": 9.65522, - "98": 9.69868, - "99": 9.75619, - "100": 9.63837 + "95": 9.75729, + "96": 9.73755, + "97": 9.65533, + "98": 9.69873, + "99": 9.75617, + "100": 9.6384 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1978.0, - "2": 1895.0, - "3": 1998.0, - "4": 1966.0, - "5": 1875.0, - "6": 1861.0, - "7": 2170.0, - "8": 1843.0, - "9": 1944.0, - "10": 1981.0, - "11": 1941.0, - "12": 1919.0, - "13": 1998.0, - "14": 2075.0, - "15": 1740.0, - "16": 1985.0, - "17": 2016.0, - "18": 2009.0, - "19": 1899.0, - "20": 1950.0, - "21": 2023.0, - "22": 2029.0, - "23": 1864.0, - "24": 2101.0, - "25": 1890.0, - "26": 1878.0, - "27": 2006.0, - "28": 1946.0, - "29": 2100.0, - "30": 1930.0, - "31": 1962.0, - "32": 1987.0, - "33": 2055.0, - "34": 2071.0, - "35": 2105.0, - "36": 2062.0, - "37": 2260.0, - "38": 2200.0, - "39": 2203.0, - "40": 2429.0, - "41": 2313.0, - "42": 2009.0, - "43": 2315.0, - "44": 2252.0, - "45": 2617.0, - "46": 2370.0, - "47": 2401.0, - "48": 2580.0, - "49": 2660.0, - "50": 2554.0, - "51": 2637.0, - "52": 2630.0, - "53": 2566.0, - "54": 2752.0, - "55": 2425.0, - "56": 2821.0, - "57": 2290.0, - "58": 3193.0, - "59": 2711.0, - "60": 2823.0, - "61": 2724.0, - "62": 2944.0, - "63": 3036.0, - "64": 3205.0, - "65": 2564.0, - "66": 2898.0, - "67": 3355.0, - "68": 3084.0, - "69": 3068.0, - "70": 2941.0, - "71": 2941.0, - "72": 3184.0, - "73": 3234.0, - "74": 3085.0, - "75": 3120.0, - "76": 3286.0, - "77": 3099.0, - "78": 3156.0, - "79": 2924.0, - "80": 3054.0, - "81": 3184.0, - "82": 3181.0, - "83": 2900.0, - "84": 2870.0, - "85": 2927.0, - "86": 3155.0, - "87": 3329.0, - "88": 3293.0, - "89": 3022.0, - "90": 3468.0, - "91": 2884.0, - "92": 2866.0, - "93": 3052.0, - "94": 3389.0, - "95": 3208.0, - "96": 3251.0, - "97": 3320.0, - "98": 3483.0, - "99": 3022.0, - "100": 3058.0 + "1": 2002.0, + "2": 1836.0, + "3": 1999.0, + "4": 2016.0, + "5": 1979.0, + "6": 1886.0, + "7": 2156.0, + "8": 1876.0, + "9": 1888.0, + "10": 2089.0, + "11": 1977.0, + "12": 1914.0, + "13": 2062.0, + "14": 2158.0, + "15": 1776.0, + "16": 2023.0, + "17": 2071.0, + "18": 2037.0, + "19": 1910.0, + "20": 1877.0, + "21": 2070.0, + "22": 2000.0, + "23": 1827.0, + "24": 1955.0, + "25": 1929.0, + "26": 1983.0, + "27": 2025.0, + "28": 1908.0, + "29": 2104.0, + "30": 1924.0, + "31": 1992.0, + "32": 2010.0, + "33": 1999.0, + "34": 2167.0, + "35": 2124.0, + "36": 2153.0, + "37": 2278.0, + "38": 2169.0, + "39": 2249.0, + "40": 2403.0, + "41": 2380.0, + "42": 2181.0, + "43": 2373.0, + "44": 2225.0, + "45": 2569.0, + "46": 2340.0, + "47": 2421.0, + "48": 2506.0, + "49": 2700.0, + "50": 2597.0, + "51": 2587.0, + "52": 2682.0, + "53": 2576.0, + "54": 2762.0, + "55": 2463.0, + "56": 2721.0, + "57": 2338.0, + "58": 3163.0, + "59": 2810.0, + "60": 2899.0, + "61": 2673.0, + "62": 2957.0, + "63": 3113.0, + "64": 3081.0, + "65": 2515.0, + "66": 2833.0, + "67": 3401.0, + "68": 3042.0, + "69": 3007.0, + "70": 3051.0, + "71": 2909.0, + "72": 3091.0, + "73": 3306.0, + "74": 3079.0, + "75": 3091.0, + "76": 3238.0, + "77": 3202.0, + "78": 3225.0, + "79": 2914.0, + "80": 2992.0, + "81": 3244.0, + "82": 3245.0, + "83": 2860.0, + "84": 3002.0, + "85": 2944.0, + "86": 3275.0, + "87": 3306.0, + "88": 3234.0, + "89": 3091.0, + "90": 3319.0, + "91": 2865.0, + "92": 2913.0, + "93": 3038.0, + "94": 3317.0, + "95": 3212.0, + "96": 3275.0, + "97": 3311.0, + "98": 3360.0, + "99": 3145.0, + "100": 2942.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.85505, - "3": 1.19024, - "4": 0.84359, - "5": 0.76463, - "6": 0.87917, - "7": 0.66058, - "8": 1.20151, - "9": 0.79788, - "10": 1.24536, - "11": 0.82619, - "12": 1.1669, - "13": 0.36186, - "14": 1.36446, - "15": 0.81937, - "16": 0.67434, - "17": 1.2472, - "18": 0.88548, - "19": 1.01634, - "20": 0.9387, - "21": 0.95791, - "22": 0.91143, - "23": 0.85542, - "24": 1.45269, - "25": 0.65296, - "26": 1.10984, - "27": 0.61392, - "28": 0.83337, - "29": 1.07639, - "30": 0.87011, - "31": 0.83554, - "32": 1.61682, - "33": 0.96985, - "34": 0.23125, - "35": 1.26054, - "36": 1.13629, - "37": 0.83452, - "38": 0.53082, - "39": 1.11374, - "40": 1.37277, - "41": 0.91611, - "42": 0.94958, - "43": 0.65567, - "44": 1.24464, - "45": 0.84205, - "46": 0.82381, - "47": 1.10749, - "48": 0.82344, - "49": 0.72087, - "50": 1.07304, - "51": 0.72789, - "52": 1.08556, - "53": 0.90301, - "54": 0.94685, - "55": 1.05426, - "56": 1.08655, - "57": 0.76066, - "58": 1.26044, - "59": 0.8244, - "60": 0.58204, - "61": 1.11386, - "62": 0.94042, - "63": 1.28285, - "64": 1.15642, - "65": 0.65714, - "66": 0.98269, - "67": 0.97012, - "68": 0.818, - "69": 1.08371, - "70": 0.77491, - "71": 1.17647, - "72": 1.18433, - "73": 0.69762, - "74": 0.98057, - "75": 0.97667, - "76": 0.83648, - "77": 0.74425, - "78": 0.85482, - "79": 1.36084, - "80": 1.01719, - "81": 1.01442, - "82": 1.03397, - "83": 0.95018, - "84": 0.73442, - "85": 0.74985, - "86": 1.22632, - "87": 0.81708, - "88": 1.11293, - "89": 1.18558, - "90": 1.05032, - "91": 0.93457, - "92": 1.52191, - "93": 1.15913, - "94": 1.25951, - "95": 1.06046, - "96": 0.86132, - "97": 0.84724, - "98": 1.10731, - "99": 1.51862, - "100": 0.84245 + "2": 3.89333, + "3": 1.09529, + "4": 0.9404, + "5": 0.63677, + "6": 0.96091, + "7": 0.73745, + "8": 1.02354, + "9": 1.16195, + "10": 0.68687, + "11": 0.6773, + "12": 0.82437, + "13": 0.8079, + "14": 0.88212, + "15": 0.61947, + "16": 0.79852, + "17": 0.96315, + "18": 0.96742, + "19": 0.71701, + "20": 0.87525, + "21": 0.79384, + "22": 0.87058, + "23": 0.66697, + "24": 0.71469, + "25": 0.79529, + "26": 1.0354, + "27": 0.56462, + "28": 0.79221, + "29": 0.78296, + "30": 0.40656, + "31": 0.61413, + "32": 1.03291, + "33": 1.1095, + "34": 0.32731, + "35": 1.3484, + "36": 0.71014, + "37": 0.91142, + "38": 1.01237, + "39": 1.16612, + "40": 1.4266, + "41": 1.00518, + "42": 1.04164, + "43": 0.75704, + "44": 1.21481, + "45": 1.07329, + "46": 0.7158, + "47": 0.94973, + "48": 0.85431, + "49": 0.71239, + "50": 0.87295, + "51": 0.24018, + "52": 0.90642, + "53": 1.17092, + "54": 1.30907, + "55": 1.2289, + "56": 0.77088, + "57": 0.88254, + "58": 0.73874, + "59": 1.0792, + "60": 1.05371, + "61": 0.71707, + "62": 1.35676, + "63": 0.89986, + "64": 0.97842, + "65": 0.71588, + "66": 0.98299, + "67": 0.90356, + "68": 0.70711, + "69": 1.05649, + "70": 0.99313, + "71": 1.03751, + "72": 0.8793, + "73": 0.82249, + "74": 0.60612, + "75": 0.82985, + "76": 0.97337, + "77": 1.02208, + "78": 1.09474, + "79": 1.27171, + "80": 0.90859, + "81": 0.59078, + "82": 1.11797, + "83": 0.87022, + "84": 0.71112, + "85": 0.59826, + "86": 0.82491, + "87": 1.37284, + "88": 0.81979, + "89": 0.56676, + "90": 1.1162, + "91": 0.77446, + "92": 0.95234, + "93": 0.79394, + "94": 1.2769, + "95": 0.79089, + "96": 1.16163, + "97": 0.69654, + "98": 0.692, + "99": 1.30656, + "100": 0.76053 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json index ca33000f71c..9432fcde908 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92706, - "2": 10.92279, - "3": 10.93187, - "4": 10.93517, - "5": 10.93088, - "6": 10.92314, - "7": 10.935, - "8": 10.92462, - "9": 10.92994, - "10": 10.92922, - "11": 10.91776, - "12": 10.92571, - "13": 10.91876, + "1": 10.92698, + "2": 10.9227, + "3": 10.9319, + "4": 10.93526, + "5": 10.93085, + "6": 10.92306, + "7": 10.93505, + "8": 10.92461, + "9": 10.92991, + "10": 10.92925, + "11": 10.91773, + "12": 10.92574, + "13": 10.91873, "14": 10.90866, - "15": 10.88351, - "16": 10.87865, - "17": 10.88879, - "18": 10.8647, - "19": 10.8723, - "20": 10.78245, - "21": 10.78372, - "22": 10.76888, - "23": 10.7643, - "24": 10.7292, - "25": 10.73568, - "26": 10.71326, - "27": 10.68182, - "28": 10.60192, - "29": 10.58355, - "30": 10.55748, - "31": 10.56241, - "32": 10.53489, + "15": 10.88358, + "16": 10.87866, + "17": 10.88886, + "18": 10.86471, + "19": 10.87233, + "20": 10.78255, + "21": 10.78375, + "22": 10.769, + "23": 10.76442, + "24": 10.72927, + "25": 10.73567, + "26": 10.71336, + "27": 10.6818, + "28": 10.60193, + "29": 10.58356, + "30": 10.55753, + "31": 10.5625, + "32": 10.5349, "33": 10.50278, - "34": 10.46238, - "35": 10.47709, - "36": 10.4569, - "37": 10.41867, - "38": 10.41656, - "39": 10.38139, - "40": 10.37436, - "41": 10.3427, - "42": 10.33072, - "43": 10.3091, - "44": 10.27515, - "45": 10.30082, - "46": 10.25975, - "47": 10.2376, - "48": 10.19352, - "49": 10.18477, - "50": 10.1973, - "51": 10.19463, - "52": 10.14642, - "53": 10.15199, - "54": 10.11839, - "55": 10.0834, - "56": 10.12273, - "57": 10.10527, - "58": 10.12069, - "59": 10.06308, - "60": 10.08102, - "61": 10.0388, - "62": 10.00454, - "63": 10.07807, - "64": 10.03218, - "65": 10.0031, - "66": 10.03544, - "67": 10.00622, - "68": 9.96933, - "69": 9.99099, - "70": 9.97868, - "71": 9.99799, - "72": 9.97581, - "73": 9.9659, - "74": 9.95983, - "75": 9.92572, - "76": 9.96561, - "77": 9.95586, - "78": 9.90228, - "79": 9.91035, - "80": 9.92617, - "81": 9.94443, - "82": 9.88905, - "83": 9.84927, - "84": 9.78863, - "85": 9.77638, - "86": 9.8798, - "87": 9.91274, - "88": 9.88586, - "89": 9.82377, - "90": 9.81124, - "91": 9.82082, - "92": 9.81006, - "93": 9.75214, - "94": 9.82524, - "95": 9.81926, - "96": 9.80583, - "97": 9.74352, - "98": 9.77347, - "99": 9.8164, - "100": 9.71043 + "34": 10.46248, + "35": 10.47715, + "36": 10.45697, + "37": 10.4187, + "38": 10.41664, + "39": 10.38143, + "40": 10.37437, + "41": 10.34276, + "42": 10.33077, + "43": 10.30916, + "44": 10.2752, + "45": 10.30087, + "46": 10.25977, + "47": 10.23764, + "48": 10.19356, + "49": 10.18482, + "50": 10.19738, + "51": 10.1947, + "52": 10.14645, + "53": 10.15204, + "54": 10.11846, + "55": 10.08345, + "56": 10.12277, + "57": 10.1054, + "58": 10.12074, + "59": 10.06313, + "60": 10.08108, + "61": 10.03883, + "62": 10.00458, + "63": 10.07813, + "64": 10.03224, + "65": 10.00315, + "66": 10.03547, + "67": 10.00629, + "68": 9.96938, + "69": 9.99101, + "70": 9.97878, + "71": 9.99803, + "72": 9.97585, + "73": 9.96598, + "74": 9.95985, + "75": 9.9258, + "76": 9.96563, + "77": 9.95588, + "78": 9.90231, + "79": 9.91042, + "80": 9.9262, + "81": 9.9445, + "82": 9.88904, + "83": 9.84931, + "84": 9.78866, + "85": 9.7764, + "86": 9.87986, + "87": 9.91273, + "88": 9.88591, + "89": 9.8238, + "90": 9.81128, + "91": 9.82089, + "92": 9.81009, + "93": 9.75217, + "94": 9.82529, + "95": 9.81925, + "96": 9.80588, + "97": 9.7436, + "98": 9.77346, + "99": 9.81647, + "100": 9.71052 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1734.0, - "2": 1786.0, - "3": 1803.0, - "4": 1688.0, - "5": 1754.0, - "6": 1745.0, - "7": 1999.0, - "8": 1746.0, - "9": 1851.0, - "10": 1758.0, - "11": 1707.0, - "12": 1670.0, - "13": 1808.0, - "14": 1971.0, - "15": 1576.0, - "16": 1798.0, - "17": 1775.0, - "18": 1779.0, - "19": 1735.0, - "20": 1721.0, - "21": 1737.0, - "22": 1741.0, - "23": 1725.0, - "24": 1771.0, - "25": 1679.0, - "26": 1785.0, - "27": 1814.0, - "28": 1773.0, - "29": 1950.0, - "30": 1873.0, - "31": 1960.0, - "32": 1922.0, - "33": 1784.0, - "34": 2086.0, - "35": 1969.0, - "36": 2017.0, - "37": 2126.0, - "38": 2057.0, - "39": 2201.0, - "40": 2326.0, - "41": 2430.0, - "42": 2017.0, - "43": 2418.0, - "44": 2153.0, - "45": 2578.0, - "46": 2194.0, - "47": 2454.0, - "48": 2644.0, - "49": 2878.0, - "50": 2585.0, - "51": 2589.0, - "52": 2772.0, - "53": 2620.0, - "54": 2882.0, - "55": 2562.0, - "56": 2638.0, - "57": 2236.0, - "58": 3618.0, - "59": 2907.0, - "60": 3104.0, - "61": 2695.0, + "1": 1760.0, + "2": 1718.0, + "3": 1911.0, + "4": 1680.0, + "5": 1724.0, + "6": 1726.0, + "7": 1938.0, + "8": 1714.0, + "9": 1777.0, + "10": 1782.0, + "11": 1731.0, + "12": 1732.0, + "13": 1901.0, + "14": 1879.0, + "15": 1638.0, + "16": 1777.0, + "17": 1870.0, + "18": 1819.0, + "19": 1636.0, + "20": 1783.0, + "21": 1782.0, + "22": 1829.0, + "23": 1680.0, + "24": 1788.0, + "25": 1765.0, + "26": 1787.0, + "27": 1775.0, + "28": 1761.0, + "29": 1969.0, + "30": 1913.0, + "31": 2024.0, + "32": 1965.0, + "33": 1939.0, + "34": 2098.0, + "35": 2015.0, + "36": 1999.0, + "37": 2184.0, + "38": 2127.0, + "39": 2269.0, + "40": 2213.0, + "41": 2434.0, + "42": 2011.0, + "43": 2385.0, + "44": 2183.0, + "45": 2492.0, + "46": 2353.0, + "47": 2532.0, + "48": 2550.0, + "49": 2889.0, + "50": 2573.0, + "51": 2495.0, + "52": 2757.0, + "53": 2559.0, + "54": 2871.0, + "55": 2674.0, + "56": 2624.0, + "57": 2119.0, + "58": 3445.0, + "59": 2871.0, + "60": 3009.0, + "61": 2660.0, "62": 3290.0, - "63": 3374.0, - "64": 3512.0, - "65": 2598.0, - "66": 3025.0, - "67": 3825.0, - "68": 3285.0, - "69": 2935.0, - "70": 3254.0, - "71": 3251.0, - "72": 2977.0, - "73": 3407.0, - "74": 3152.0, - "75": 3256.0, - "76": 3120.0, - "77": 3711.0, - "78": 3219.0, - "79": 3165.0, - "80": 3018.0, - "81": 3312.0, - "82": 3215.0, - "83": 3166.0, - "84": 2938.0, - "85": 2700.0, - "86": 3277.0, - "87": 2976.0, - "88": 3161.0, - "89": 2993.0, - "90": 3642.0, - "91": 3049.0, - "92": 2919.0, - "93": 3041.0, - "94": 3165.0, - "95": 3290.0, - "96": 3514.0, - "97": 3669.0, - "98": 3398.0, - "99": 3337.0, - "100": 3183.0 + "63": 3225.0, + "64": 3763.0, + "65": 2612.0, + "66": 3044.0, + "67": 3757.0, + "68": 3424.0, + "69": 2882.0, + "70": 3216.0, + "71": 3121.0, + "72": 3021.0, + "73": 3434.0, + "74": 3146.0, + "75": 3203.0, + "76": 3066.0, + "77": 3704.0, + "78": 3253.0, + "79": 3121.0, + "80": 3113.0, + "81": 3417.0, + "82": 3220.0, + "83": 3187.0, + "84": 2939.0, + "85": 2734.0, + "86": 3071.0, + "87": 3029.0, + "88": 3078.0, + "89": 3030.0, + "90": 3639.0, + "91": 3109.0, + "92": 2912.0, + "93": 3082.0, + "94": 3308.0, + "95": 3517.0, + "96": 3691.0, + "97": 3679.0, + "98": 3363.0, + "99": 3332.0, + "100": 3322.0 } }, "mem-allocated-bytes": { @@ -325,7 +325,7 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1939785728.0, + "1": 1938737152.0, "2": 2222434304.0, "3": 2222434304.0, "4": 2222434304.0, @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.6207, - "3": 0.15039, - "4": 0.13396, - "5": 0.13283, - "6": 0.13232, - "7": 0.1325, - "8": 0.13239, - "9": 0.13223, - "10": 0.13388, - "11": 0.13411, - "12": 0.1337, - "13": 0.13259, - "14": 0.13256, - "15": 0.13335, - "16": 0.13319, - "17": 0.13327, - "18": 0.13428, - "19": 0.13272, - "20": 0.13286, - "21": 0.1329, - "22": 0.13325, - "23": 0.13108, - "24": 0.13182, - "25": 0.13197, - "26": 0.13141, - "27": 0.13269, - "28": 0.13214, - "29": 0.1318, - "30": 0.13273, - "31": 0.13264, - "32": 0.13249, - "33": 0.13301, - "34": 0.13397, - "35": 0.13449, - "36": 0.13312, - "37": 0.13394, - "38": 0.13268, - "39": 0.13296, - "40": 0.13296, - "41": 0.13339, - "42": 0.13287, - "43": 0.13309, - "44": 0.13288, - "45": 0.13284, - "46": 0.1331, - "47": 0.13267, - "48": 0.13415, - "49": 0.13433, - "50": 0.13334, - "51": 0.30009, - "52": 0.16347, - "53": 0.13657, - "54": 0.13308, - "55": 0.13255, - "56": 0.13185, - "57": 0.13186, - "58": 0.13237, - "59": 0.1323, - "60": 0.13212, - "61": 0.13313, - "62": 0.13295, - "63": 0.13259, - "64": 0.1328, - "65": 0.13259, - "66": 0.13266, - "67": 0.13246, - "68": 0.1324, - "69": 0.13728, - "70": 0.1325, - "71": 0.13274, - "72": 0.13247, - "73": 0.13252, - "74": 0.13306, - "75": 0.1333, - "76": 0.13367, - "77": 0.13294, - "78": 0.13322, - "79": 0.13299, - "80": 0.13227, - "81": 0.13331, - "82": 0.13315, - "83": 0.13287, - "84": 0.13344, - "85": 0.13348, - "86": 0.13241, - "87": 0.13291, - "88": 0.1329, - "89": 0.13301, - "90": 0.13279, - "91": 0.13519, - "92": 0.13293, - "93": 0.1323, - "94": 0.13273, - "95": 0.13252, - "96": 0.13333, - "97": 0.13261, - "98": 0.13246, - "99": 0.1324, - "100": 0.13332 + "2": 7.66352, + "3": 0.16745, + "4": 0.11219, + "5": 0.11103, + "6": 0.10822, + "7": 0.10924, + "8": 0.11006, + "9": 0.10962, + "10": 0.10973, + "11": 0.1099, + "12": 0.11265, + "13": 0.10804, + "14": 0.11186, + "15": 0.11277, + "16": 0.11292, + "17": 0.10952, + "18": 0.10913, + "19": 0.11017, + "20": 0.11238, + "21": 0.1113, + "22": 0.10953, + "23": 0.11027, + "24": 0.11123, + "25": 0.10982, + "26": 0.11025, + "27": 0.11005, + "28": 0.1111, + "29": 0.11164, + "30": 0.11086, + "31": 0.11084, + "32": 0.11109, + "33": 0.11034, + "34": 0.10832, + "35": 0.10811, + "36": 0.10879, + "37": 0.11073, + "38": 0.112, + "39": 0.11257, + "40": 0.11108, + "41": 0.1102, + "42": 0.11221, + "43": 0.11007, + "44": 0.11002, + "45": 0.11006, + "46": 0.10973, + "47": 0.11135, + "48": 0.11085, + "49": 0.10894, + "50": 0.10773, + "51": 0.27983, + "52": 0.15473, + "53": 0.11524, + "54": 0.11054, + "55": 0.11097, + "56": 0.11076, + "57": 0.11156, + "58": 0.11141, + "59": 0.11383, + "60": 0.11263, + "61": 0.26658, + "62": 0.11025, + "63": 0.11139, + "64": 0.23636, + "65": 0.11051, + "66": 0.11223, + "67": 0.11288, + "68": 0.11313, + "69": 0.11142, + "70": 0.1103, + "71": 0.11074, + "72": 0.10928, + "73": 0.11282, + "74": 0.11425, + "75": 0.11317, + "76": 0.11042, + "77": 0.11188, + "78": 0.11464, + "79": 0.10979, + "80": 0.11136, + "81": 0.11078, + "82": 0.10972, + "83": 0.11807, + "84": 0.11019, + "85": 0.11154, + "86": 0.11127, + "87": 0.1136, + "88": 0.11104, + "89": 0.11438, + "90": 0.11202, + "91": 0.11184, + "92": 0.11243, + "93": 0.1111, + "94": 0.11183, + "95": 0.10891, + "96": 0.10911, + "97": 0.11012, + "98": 0.1095, + "99": 0.10941, + "100": 0.11136 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json index 02f836290c2..9c50e30407f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95939, - "2": 10.94561, - "3": 10.94918, + "1": 10.95945, + "2": 10.94558, + "3": 10.94922, "4": 10.94704, - "5": 10.9559, - "6": 10.95226, - "7": 10.95018, - "8": 10.95679, - "9": 10.94257, - "10": 10.94725, - "11": 10.9355, + "5": 10.95589, + "6": 10.95218, + "7": 10.95016, + "8": 10.95689, + "9": 10.94261, + "10": 10.94724, + "11": 10.93546, "12": 10.94741, - "13": 10.9199, - "14": 10.92082, - "15": 10.90466, - "16": 10.89239, - "17": 10.89288, - "18": 10.89246, - "19": 10.87971, - "20": 10.80737, - "21": 10.78841, - "22": 10.77824, - "23": 10.78255, - "24": 10.75943, - "25": 10.75771, - "26": 10.73464, - "27": 10.69864, - "28": 10.61058, - "29": 10.57945, - "30": 10.56536, - "31": 10.56477, - "32": 10.55151, - "33": 10.52212, - "34": 10.47131, - "35": 10.48348, - "36": 10.46229, - "37": 10.42487, - "38": 10.42072, - "39": 10.39099, - "40": 10.37297, - "41": 10.35274, - "42": 10.35024, - "43": 10.31689, - "44": 10.30513, - "45": 10.2969, - "46": 10.26609, - "47": 10.25516, - "48": 10.21044, - "49": 10.20228, - "50": 10.21713, - "51": 10.21029, - "52": 10.15829, - "53": 10.15994, - "54": 10.13462, - "55": 10.11077, - "56": 10.12861, - "57": 10.11008, - "58": 10.12175, - "59": 10.06878, - "60": 10.08995, - "61": 10.04525, - "62": 10.01558, - "63": 10.08025, - "64": 10.03236, - "65": 10.00171, - "66": 10.04585, - "67": 10.01598, - "68": 9.98431, - "69": 9.99678, - "70": 9.98004, - "71": 10.00125, - "72": 9.98844, + "13": 10.91988, + "14": 10.92077, + "15": 10.90467, + "16": 10.8924, + "17": 10.89279, + "18": 10.89248, + "19": 10.87973, + "20": 10.80736, + "21": 10.78846, + "22": 10.77833, + "23": 10.78262, + "24": 10.75945, + "25": 10.75778, + "26": 10.73468, + "27": 10.69867, + "28": 10.61066, + "29": 10.57956, + "30": 10.56539, + "31": 10.56486, + "32": 10.55161, + "33": 10.52219, + "34": 10.47136, + "35": 10.48355, + "36": 10.46232, + "37": 10.42496, + "38": 10.42078, + "39": 10.39103, + "40": 10.37295, + "41": 10.35282, + "42": 10.35034, + "43": 10.31696, + "44": 10.30514, + "45": 10.29693, + "46": 10.26616, + "47": 10.2552, + "48": 10.21043, + "49": 10.2023, + "50": 10.21715, + "51": 10.21037, + "52": 10.1583, + "53": 10.15998, + "54": 10.13463, + "55": 10.11084, + "56": 10.12869, + "57": 10.11006, + "58": 10.12183, + "59": 10.06883, + "60": 10.09, + "61": 10.0453, + "62": 10.01564, + "63": 10.08031, + "64": 10.03238, + "65": 10.00175, + "66": 10.04593, + "67": 10.01595, + "68": 9.98437, + "69": 9.99683, + "70": 9.98003, + "71": 10.00129, + "72": 9.98841, "73": 9.97265, - "74": 9.96397, - "75": 9.92908, - "76": 9.95782, - "77": 9.95865, - "78": 9.91024, - "79": 9.9085, - "80": 9.92434, - "81": 9.94839, - "82": 9.8887, - "83": 9.85304, - "84": 9.79711, - "85": 9.7777, - "86": 9.88374, - "87": 9.90613, - "88": 9.87919, - "89": 9.81825, - "90": 9.81548, - "91": 9.81755, - "92": 9.81598, - "93": 9.74136, - "94": 9.82061, - "95": 9.80831, - "96": 9.79427, - "97": 9.73959, - "98": 9.76307, - "99": 9.81733, - "100": 9.70258 + "74": 9.96398, + "75": 9.92912, + "76": 9.95785, + "77": 9.95872, + "78": 9.91029, + "79": 9.90855, + "80": 9.92441, + "81": 9.94842, + "82": 9.88868, + "83": 9.85308, + "84": 9.79712, + "85": 9.77773, + "86": 9.88383, + "87": 9.90616, + "88": 9.87921, + "89": 9.8183, + "90": 9.81557, + "91": 9.81758, + "92": 9.81601, + "93": 9.74139, + "94": 9.82066, + "95": 9.80836, + "96": 9.79432, + "97": 9.73962, + "98": 9.76311, + "99": 9.81735, + "100": 9.70263 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1688.0, - "2": 1724.0, - "3": 1783.0, - "4": 1601.0, - "5": 1744.0, - "6": 1688.0, - "7": 1868.0, - "8": 1702.0, - "9": 1745.0, - "10": 1683.0, - "11": 1563.0, - "12": 1636.0, - "13": 1778.0, - "14": 1773.0, - "15": 1694.0, + "1": 1757.0, + "2": 1634.0, + "3": 1778.0, + "4": 1689.0, + "5": 1718.0, + "6": 1665.0, + "7": 1932.0, + "8": 1688.0, + "9": 1691.0, + "10": 1675.0, + "11": 1611.0, + "12": 1641.0, + "13": 1752.0, + "14": 1772.0, + "15": 1704.0, "16": 1707.0, - "17": 1779.0, - "18": 1679.0, - "19": 1620.0, - "20": 1646.0, - "21": 1760.0, - "22": 1708.0, - "23": 1621.0, - "24": 1776.0, - "25": 1673.0, - "26": 1769.0, - "27": 1866.0, - "28": 1756.0, - "29": 1903.0, - "30": 1877.0, - "31": 2055.0, - "32": 2002.0, - "33": 2047.0, - "34": 1991.0, - "35": 2000.0, - "36": 1885.0, - "37": 2216.0, - "38": 2158.0, - "39": 2212.0, - "40": 2320.0, - "41": 2350.0, - "42": 2074.0, - "43": 2261.0, - "44": 2201.0, - "45": 2469.0, - "46": 2322.0, - "47": 2456.0, - "48": 2498.0, - "49": 2759.0, - "50": 2514.0, - "51": 2446.0, - "52": 2598.0, - "53": 2592.0, - "54": 2706.0, - "55": 2471.0, - "56": 2631.0, - "57": 2252.0, - "58": 3577.0, - "59": 2890.0, - "60": 2988.0, - "61": 2621.0, - "62": 3002.0, - "63": 3226.0, - "64": 3493.0, - "65": 2609.0, - "66": 2957.0, - "67": 3658.0, - "68": 3215.0, - "69": 2894.0, - "70": 3265.0, - "71": 3065.0, - "72": 2937.0, - "73": 3491.0, - "74": 3258.0, - "75": 2911.0, - "76": 3278.0, - "77": 3785.0, - "78": 3230.0, - "79": 3147.0, - "80": 3018.0, - "81": 3468.0, - "82": 3034.0, - "83": 3111.0, - "84": 2921.0, - "85": 2608.0, - "86": 2990.0, - "87": 2705.0, - "88": 3045.0, - "89": 3038.0, - "90": 3834.0, - "91": 3046.0, - "92": 3063.0, - "93": 2974.0, - "94": 3253.0, - "95": 3215.0, - "96": 3509.0, - "97": 3520.0, - "98": 3331.0, - "99": 3062.0, - "100": 2906.0 + "17": 1774.0, + "18": 1653.0, + "19": 1654.0, + "20": 1674.0, + "21": 1884.0, + "22": 1717.0, + "23": 1682.0, + "24": 1842.0, + "25": 1672.0, + "26": 1827.0, + "27": 1781.0, + "28": 1787.0, + "29": 1874.0, + "30": 1756.0, + "31": 2047.0, + "32": 2000.0, + "33": 2016.0, + "34": 1927.0, + "35": 2059.0, + "36": 1953.0, + "37": 2219.0, + "38": 2079.0, + "39": 2287.0, + "40": 2267.0, + "41": 2353.0, + "42": 1984.0, + "43": 2366.0, + "44": 2237.0, + "45": 2406.0, + "46": 2437.0, + "47": 2490.0, + "48": 2610.0, + "49": 2824.0, + "50": 2462.0, + "51": 2550.0, + "52": 2650.0, + "53": 2604.0, + "54": 2782.0, + "55": 2379.0, + "56": 2607.0, + "57": 2254.0, + "58": 3543.0, + "59": 2766.0, + "60": 2903.0, + "61": 2703.0, + "62": 3189.0, + "63": 3372.0, + "64": 3554.0, + "65": 2665.0, + "66": 2967.0, + "67": 3732.0, + "68": 3289.0, + "69": 2882.0, + "70": 3230.0, + "71": 3124.0, + "72": 2888.0, + "73": 3497.0, + "74": 3070.0, + "75": 2934.0, + "76": 3294.0, + "77": 3651.0, + "78": 3172.0, + "79": 3231.0, + "80": 3072.0, + "81": 3623.0, + "82": 3038.0, + "83": 3112.0, + "84": 2955.0, + "85": 2647.0, + "86": 2960.0, + "87": 2632.0, + "88": 2986.0, + "89": 3002.0, + "90": 3706.0, + "91": 2999.0, + "92": 2941.0, + "93": 2999.0, + "94": 3127.0, + "95": 3168.0, + "96": 3607.0, + "97": 3401.0, + "98": 3405.0, + "99": 2933.0, + "100": 3025.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 746194432.0, - "2": 746194432.0, - "3": 746194432.0, - "4": 746194432.0, - "5": 746194432.0, - "6": 746194432.0, - "7": 746194432.0, - "8": 746194432.0, - "9": 746194432.0, - "10": 746194432.0, - "11": 746194432.0, - "12": 746194432.0, - "13": 746194432.0, - "14": 746194432.0, - "15": 746194432.0, - "16": 746194432.0, - "17": 746194432.0, - "18": 746194432.0, - "19": 746194432.0, - "20": 746194432.0, - "21": 746194432.0, - "22": 746194432.0, - "23": 746194432.0, - "24": 746194432.0, - "25": 746194432.0, - "26": 746194432.0, - "27": 746194432.0, - "28": 746194432.0, - "29": 746194432.0, - "30": 746194432.0, - "31": 746194432.0, - "32": 746194432.0, - "33": 746194432.0, - "34": 746194432.0, - "35": 746194432.0, - "36": 746194432.0, - "37": 746194432.0, - "38": 746194432.0, - "39": 746194432.0, - "40": 746194432.0, - "41": 746194432.0, - "42": 746194432.0, - "43": 746194432.0, - "44": 746194432.0, - "45": 746194432.0, - "46": 746194432.0, - "47": 746194432.0, - "48": 746194432.0, - "49": 746194432.0, - "50": 746194432.0, - "51": 746194432.0, - "52": 746194432.0, - "53": 746194432.0, - "54": 746194432.0, - "55": 746194432.0, - "56": 746194432.0, - "57": 746194432.0, - "58": 746194432.0, - "59": 746194432.0, - "60": 746194432.0, - "61": 746194432.0, - "62": 746194432.0, - "63": 746194432.0, - "64": 746194432.0, - "65": 746194432.0, - "66": 746194432.0, - "67": 746194432.0, - "68": 746194432.0, - "69": 746194432.0, - "70": 746194432.0, - "71": 746194432.0, - "72": 746194432.0, - "73": 746194432.0, - "74": 746194432.0, - "75": 746194432.0, - "76": 746194432.0, - "77": 746194432.0, - "78": 746194432.0, - "79": 746194432.0, - "80": 746194432.0, - "81": 746194432.0, - "82": 746194432.0, - "83": 746194432.0, - "84": 746194432.0, - "85": 746194432.0, - "86": 746194432.0, - "87": 746194432.0, - "88": 746194432.0, - "89": 746194432.0, - "90": 746194432.0, - "91": 746194432.0, - "92": 746194432.0, - "93": 746194432.0, - "94": 746194432.0, - "95": 746194432.0, - "96": 746194432.0, - "97": 746194432.0, - "98": 746194432.0, - "99": 746194432.0, - "100": 746194432.0 + "1": 747244032.0, + "2": 747244032.0, + "3": 747244032.0, + "4": 747244032.0, + "5": 747244032.0, + "6": 747244032.0, + "7": 747244032.0, + "8": 747244032.0, + "9": 747244032.0, + "10": 747244032.0, + "11": 747244032.0, + "12": 747244032.0, + "13": 747244032.0, + "14": 747244032.0, + "15": 747244032.0, + "16": 747244032.0, + "17": 747244032.0, + "18": 747244032.0, + "19": 747244032.0, + "20": 747244032.0, + "21": 747244032.0, + "22": 747244032.0, + "23": 747244032.0, + "24": 747244032.0, + "25": 747244032.0, + "26": 747244032.0, + "27": 747244032.0, + "28": 747244032.0, + "29": 747244032.0, + "30": 747244032.0, + "31": 747244032.0, + "32": 747244032.0, + "33": 747244032.0, + "34": 747244032.0, + "35": 747244032.0, + "36": 747244032.0, + "37": 747244032.0, + "38": 747244032.0, + "39": 747244032.0, + "40": 747244032.0, + "41": 747244032.0, + "42": 747244032.0, + "43": 747244032.0, + "44": 747244032.0, + "45": 747244032.0, + "46": 747244032.0, + "47": 747244032.0, + "48": 747244032.0, + "49": 747244032.0, + "50": 747244032.0, + "51": 747244032.0, + "52": 747244032.0, + "53": 747244032.0, + "54": 747244032.0, + "55": 747244032.0, + "56": 747244032.0, + "57": 747244032.0, + "58": 747244032.0, + "59": 747244032.0, + "60": 747244032.0, + "61": 747244032.0, + "62": 747244032.0, + "63": 747244032.0, + "64": 747244032.0, + "65": 747244032.0, + "66": 747244032.0, + "67": 747244032.0, + "68": 747244032.0, + "69": 747244032.0, + "70": 747244032.0, + "71": 747244032.0, + "72": 747244032.0, + "73": 747244032.0, + "74": 747244032.0, + "75": 747244032.0, + "76": 747244032.0, + "77": 747244032.0, + "78": 747244032.0, + "79": 747244032.0, + "80": 747244032.0, + "81": 747244032.0, + "82": 747244032.0, + "83": 747244032.0, + "84": 747244032.0, + "85": 747244032.0, + "86": 747244032.0, + "87": 747244032.0, + "88": 747244032.0, + "89": 747244032.0, + "90": 747244032.0, + "91": 747244032.0, + "92": 747244032.0, + "93": 747244032.0, + "94": 747244032.0, + "95": 747244032.0, + "96": 747244032.0, + "97": 747244032.0, + "98": 747244032.0, + "99": 747244032.0, + "100": 747244032.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1927202816.0, - "2": 2209851392.0, - "3": 2209851392.0, - "4": 2209851392.0, - "5": 2209851392.0, - "6": 2209851392.0, - "7": 2209851392.0, - "8": 2209851392.0, - "9": 2209851392.0, - "10": 2209851392.0, - "11": 2209851392.0, - "12": 2209851392.0, - "13": 2209851392.0, - "14": 2209851392.0, - "15": 2209851392.0, - "16": 2209851392.0, - "17": 2209851392.0, - "18": 2209851392.0, - "19": 2209851392.0, - "20": 2209851392.0, - "21": 2209851392.0, - "22": 2209851392.0, - "23": 2209851392.0, - "24": 2209851392.0, - "25": 2209851392.0, - "26": 2209851392.0, - "27": 2209851392.0, - "28": 2209851392.0, - "29": 2209851392.0, - "30": 2209851392.0, - "31": 2209851392.0, - "32": 2209851392.0, - "33": 2209851392.0, - "34": 2209851392.0, - "35": 2209851392.0, - "36": 2209851392.0, - "37": 2209851392.0, - "38": 2209851392.0, - "39": 2209851392.0, - "40": 2209851392.0, - "41": 2209851392.0, - "42": 2209851392.0, - "43": 2209851392.0, - "44": 2209851392.0, - "45": 2209851392.0, - "46": 2209851392.0, - "47": 2209851392.0, - "48": 2209851392.0, - "49": 2209851392.0, - "50": 2209851392.0, - "51": 2209851392.0, - "52": 2209851392.0, - "53": 2209851392.0, - "54": 2209851392.0, - "55": 2209851392.0, - "56": 2209851392.0, - "57": 2209851392.0, - "58": 2209851392.0, - "59": 2209851392.0, - "60": 2209851392.0, - "61": 2209851392.0, - "62": 2209851392.0, - "63": 2209851392.0, - "64": 2209851392.0, - "65": 2209851392.0, - "66": 2209851392.0, - "67": 2209851392.0, - "68": 2209851392.0, - "69": 2209851392.0, - "70": 2209851392.0, - "71": 2209851392.0, - "72": 2209851392.0, - "73": 2209851392.0, - "74": 2209851392.0, - "75": 2209851392.0, - "76": 2209851392.0, - "77": 2209851392.0, - "78": 2209851392.0, - "79": 2209851392.0, - "80": 2209851392.0, - "81": 2209851392.0, - "82": 2209851392.0, - "83": 2209851392.0, - "84": 2209851392.0, - "85": 2209851392.0, - "86": 2209851392.0, - "87": 2209851392.0, - "88": 2209851392.0, - "89": 2209851392.0, - "90": 2209851392.0, - "91": 2209851392.0, - "92": 2209851392.0, - "93": 2209851392.0, - "94": 2209851392.0, - "95": 2209851392.0, - "96": 2209851392.0, - "97": 2209851392.0, - "98": 2209851392.0, - "99": 2209851392.0, - "100": 2209851392.0 + "2": 2211948544.0, + "3": 2211948544.0, + "4": 2211948544.0, + "5": 2211948544.0, + "6": 2211948544.0, + "7": 2211948544.0, + "8": 2211948544.0, + "9": 2211948544.0, + "10": 2211948544.0, + "11": 2211948544.0, + "12": 2211948544.0, + "13": 2211948544.0, + "14": 2211948544.0, + "15": 2211948544.0, + "16": 2211948544.0, + "17": 2211948544.0, + "18": 2211948544.0, + "19": 2211948544.0, + "20": 2211948544.0, + "21": 2211948544.0, + "22": 2211948544.0, + "23": 2211948544.0, + "24": 2211948544.0, + "25": 2211948544.0, + "26": 2211948544.0, + "27": 2211948544.0, + "28": 2211948544.0, + "29": 2211948544.0, + "30": 2211948544.0, + "31": 2211948544.0, + "32": 2211948544.0, + "33": 2211948544.0, + "34": 2211948544.0, + "35": 2211948544.0, + "36": 2211948544.0, + "37": 2211948544.0, + "38": 2211948544.0, + "39": 2211948544.0, + "40": 2211948544.0, + "41": 2211948544.0, + "42": 2211948544.0, + "43": 2211948544.0, + "44": 2211948544.0, + "45": 2211948544.0, + "46": 2211948544.0, + "47": 2211948544.0, + "48": 2211948544.0, + "49": 2211948544.0, + "50": 2211948544.0, + "51": 2211948544.0, + "52": 2211948544.0, + "53": 2211948544.0, + "54": 2211948544.0, + "55": 2211948544.0, + "56": 2211948544.0, + "57": 2211948544.0, + "58": 2211948544.0, + "59": 2211948544.0, + "60": 2211948544.0, + "61": 2211948544.0, + "62": 2211948544.0, + "63": 2211948544.0, + "64": 2211948544.0, + "65": 2211948544.0, + "66": 2211948544.0, + "67": 2211948544.0, + "68": 2211948544.0, + "69": 2211948544.0, + "70": 2211948544.0, + "71": 2211948544.0, + "72": 2211948544.0, + "73": 2211948544.0, + "74": 2211948544.0, + "75": 2211948544.0, + "76": 2211948544.0, + "77": 2211948544.0, + "78": 2211948544.0, + "79": 2211948544.0, + "80": 2211948544.0, + "81": 2211948544.0, + "82": 2211948544.0, + "83": 2211948544.0, + "84": 2211948544.0, + "85": 2211948544.0, + "86": 2211948544.0, + "87": 2211948544.0, + "88": 2211948544.0, + "89": 2211948544.0, + "90": 2211948544.0, + "91": 2211948544.0, + "92": 2211948544.0, + "93": 2211948544.0, + "94": 2211948544.0, + "95": 2211948544.0, + "96": 2211948544.0, + "97": 2211948544.0, + "98": 2211948544.0, + "99": 2211948544.0, + "100": 2211948544.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21329, - "3": 0.12134, - "4": 0.12225, - "5": 0.11934, - "6": 0.11821, - "7": 0.11851, - "8": 0.1188, - "9": 0.11853, - "10": 0.1179, - "11": 0.11867, - "12": 0.12117, - "13": 0.11852, - "14": 0.12764, - "15": 0.11801, - "16": 0.11893, - "17": 0.11882, - "18": 0.11769, - "19": 0.12406, - "20": 0.11743, - "21": 0.11967, - "22": 0.1188, - "23": 0.11839, - "24": 0.11915, - "25": 0.121, - "26": 0.1195, - "27": 0.1205, - "28": 0.11935, - "29": 0.11885, - "30": 0.11851, - "31": 0.12896, - "32": 0.11987, - "33": 0.11824, - "34": 0.11848, - "35": 0.11849, - "36": 0.11929, - "37": 0.11878, - "38": 0.1183, - "39": 0.12395, - "40": 0.1189, - "41": 0.1193, - "42": 0.12262, - "43": 0.12068, - "44": 0.12011, - "45": 0.12003, - "46": 0.11995, - "47": 0.11961, - "48": 0.11858, - "49": 0.11919, - "50": 0.12178, - "51": 0.16041, - "52": 0.15706, - "53": 0.14154, - "54": 0.12546, - "55": 0.11932, - "56": 0.11823, - "57": 0.11686, - "58": 0.11871, - "59": 0.11852, - "60": 0.11691, - "61": 0.11768, - "62": 0.11687, - "63": 0.11776, - "64": 0.11717, - "65": 0.11979, - "66": 0.11765, - "67": 0.11776, - "68": 0.11694, - "69": 0.11643, - "70": 0.11696, - "71": 0.11721, - "72": 0.11987, - "73": 0.12135, - "74": 0.11791, - "75": 0.11866, - "76": 0.11895, - "77": 0.11941, - "78": 0.12199, - "79": 0.11977, - "80": 0.12163, - "81": 0.1193, + "2": 6.02519, + "3": 0.12256, + "4": 0.1275, + "5": 0.1216, + "6": 0.12012, + "7": 0.12011, + "8": 0.12083, + "9": 0.12222, + "10": 0.12035, + "11": 0.12206, + "12": 0.12012, + "13": 0.11853, + "14": 0.1202, + "15": 0.11959, + "16": 0.12062, + "17": 0.12436, + "18": 0.12, + "19": 0.12175, + "20": 0.12652, + "21": 0.12047, + "22": 0.11836, + "23": 0.11815, + "24": 0.11799, + "25": 0.11793, + "26": 0.11789, + "27": 0.11801, + "28": 0.1201, + "29": 0.11868, + "30": 0.11776, + "31": 0.11792, + "32": 0.11764, + "33": 0.11796, + "34": 0.11845, + "35": 0.11803, + "36": 0.11866, + "37": 0.118, + "38": 0.1182, + "39": 0.11888, + "40": 0.12222, + "41": 0.11841, + "42": 0.12081, + "43": 0.11748, + "44": 0.117, + "45": 0.11768, + "46": 0.11941, + "47": 0.12265, + "48": 0.1187, + "49": 0.11663, + "50": 0.11868, + "51": 0.16269, + "52": 0.14047, + "53": 0.12438, + "54": 0.12016, + "55": 0.12075, + "56": 0.1213, + "57": 0.12225, + "58": 0.12168, + "59": 0.1226, + "60": 0.11951, + "61": 0.12067, + "62": 0.12022, + "63": 0.12334, + "64": 0.12128, + "65": 0.11986, + "66": 0.11882, + "67": 0.12227, + "68": 0.12217, + "69": 0.12092, + "70": 0.12074, + "71": 0.11965, + "72": 0.12007, + "73": 0.12003, + "74": 0.11986, + "75": 0.11792, + "76": 0.1188, + "77": 0.11914, + "78": 0.12559, + "79": 0.11909, + "80": 0.11851, + "81": 0.11971, "82": 0.1188, - "83": 0.11977, - "84": 0.11865, - "85": 0.11971, - "86": 0.11697, - "87": 0.1178, - "88": 0.11886, - "89": 0.11687, - "90": 0.11849, - "91": 0.11798, - "92": 0.11687, - "93": 0.11878, - "94": 0.11831, - "95": 0.11793, - "96": 0.11984, - "97": 0.11979, - "98": 0.11871, - "99": 0.11966, - "100": 0.11783 + "83": 0.11899, + "84": 0.11883, + "85": 0.11895, + "86": 0.11933, + "87": 0.11813, + "88": 0.11873, + "89": 0.11937, + "90": 0.11904, + "91": 0.11785, + "92": 0.12043, + "93": 0.11865, + "94": 0.12049, + "95": 0.12021, + "96": 0.12026, + "97": 0.11972, + "98": 0.11882, + "99": 0.11943, + "100": 0.11893 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json index 4ecd4f0df18..a1a81f94b44 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92753, - "2": 10.92559, + "1": 10.92752, + "2": 10.92554, "3": 10.93114, - "4": 10.93741, - "5": 10.93016, - "6": 10.92355, - "7": 10.93863, - "8": 10.92774, - "9": 10.93076, - "10": 10.92802, - "11": 10.91411, - "12": 10.92298, - "13": 10.92014, - "14": 10.90614, + "4": 10.93727, + "5": 10.93015, + "6": 10.92346, + "7": 10.93864, + "8": 10.92777, + "9": 10.93074, + "10": 10.92801, + "11": 10.91409, + "12": 10.92299, + "13": 10.92006, + "14": 10.90618, "15": 10.88338, - "16": 10.87951, - "17": 10.88868, - "18": 10.86451, - "19": 10.87355, - "20": 10.783, - "21": 10.78553, - "22": 10.76764, - "23": 10.764, - "24": 10.72663, - "25": 10.73401, - "26": 10.71265, - "27": 10.67569, - "28": 10.6012, - "29": 10.58219, - "30": 10.55702, - "31": 10.56167, - "32": 10.53487, - "33": 10.50348, - "34": 10.46525, - "35": 10.47887, - "36": 10.45601, - "37": 10.41808, - "38": 10.4167, - "39": 10.38178, - "40": 10.37615, - "41": 10.34328, - "42": 10.33161, - "43": 10.3077, - "44": 10.27713, - "45": 10.2988, - "46": 10.25974, - "47": 10.23655, - "48": 10.19378, - "49": 10.18543, - "50": 10.19728, - "51": 10.19367, - "52": 10.14697, - "53": 10.14982, - "54": 10.11733, - "55": 10.08476, - "56": 10.12202, - "57": 10.10466, - "58": 10.11902, - "59": 10.06425, - "60": 10.08087, + "16": 10.87953, + "17": 10.88861, + "18": 10.86454, + "19": 10.87347, + "20": 10.78318, + "21": 10.78551, + "22": 10.76768, + "23": 10.76394, + "24": 10.72672, + "25": 10.734, + "26": 10.71269, + "27": 10.6757, + "28": 10.60128, + "29": 10.5823, + "30": 10.55707, + "31": 10.56165, + "32": 10.53484, + "33": 10.50353, + "34": 10.46529, + "35": 10.47894, + "36": 10.45605, + "37": 10.41812, + "38": 10.41677, + "39": 10.38189, + "40": 10.37623, + "41": 10.34336, + "42": 10.33171, + "43": 10.30768, + "44": 10.27724, + "45": 10.29884, + "46": 10.2598, + "47": 10.23664, + "48": 10.1938, + "49": 10.18553, + "50": 10.19738, + "51": 10.1937, + "52": 10.14694, + "53": 10.1499, + "54": 10.11736, + "55": 10.08481, + "56": 10.12206, + "57": 10.10473, + "58": 10.11908, + "59": 10.06432, + "60": 10.08092, "61": 10.03845, - "62": 10.00507, - "63": 10.07872, - "64": 10.03202, - "65": 10.00346, - "66": 10.03491, - "67": 10.00571, - "68": 9.97032, - "69": 9.99153, - "70": 9.97866, - "71": 9.9998, - "72": 9.97531, - "73": 9.96566, - "74": 9.95983, + "62": 10.00515, + "63": 10.07878, + "64": 10.03206, + "65": 10.00349, + "66": 10.03495, + "67": 10.00575, + "68": 9.97033, + "69": 9.99157, + "70": 9.97869, + "71": 9.99986, + "72": 9.97535, + "73": 9.96568, + "74": 9.95986, "75": 9.92549, - "76": 9.96566, + "76": 9.96565, "77": 9.95606, - "78": 9.90446, - "79": 9.91127, - "80": 9.92478, - "81": 9.94432, - "82": 9.88863, - "83": 9.85093, - "84": 9.78899, - "85": 9.77531, - "86": 9.88013, + "78": 9.9045, + "79": 9.91133, + "80": 9.92487, + "81": 9.94438, + "82": 9.88871, + "83": 9.85105, + "84": 9.78905, + "85": 9.77533, + "86": 9.88023, "87": 9.91309, - "88": 9.88639, - "89": 9.82366, - "90": 9.81326, - "91": 9.82136, - "92": 9.81011, - "93": 9.75216, - "94": 9.82571, - "95": 9.81912, - "96": 9.8055, - "97": 9.7436, - "98": 9.77248, - "99": 9.81661, - "100": 9.71126 + "88": 9.88646, + "89": 9.8237, + "90": 9.81331, + "91": 9.82141, + "92": 9.81015, + "93": 9.75217, + "94": 9.82575, + "95": 9.8191, + "96": 9.80552, + "97": 9.74366, + "98": 9.7725, + "99": 9.8167, + "100": 9.71129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1770.0, - "2": 1703.0, - "3": 1799.0, - "4": 1823.0, - "5": 1737.0, - "6": 1733.0, + "1": 1845.0, + "2": 1697.0, + "3": 1692.0, + "4": 1714.0, + "5": 1786.0, + "6": 1732.0, "7": 1952.0, - "8": 1738.0, - "9": 1687.0, - "10": 1780.0, - "11": 1673.0, - "12": 1677.0, - "13": 1724.0, - "14": 1837.0, - "15": 1570.0, - "16": 1727.0, - "17": 1788.0, - "18": 1780.0, - "19": 1660.0, - "20": 1694.0, - "21": 1732.0, - "22": 1727.0, - "23": 1703.0, - "24": 1799.0, - "25": 1588.0, - "26": 1817.0, - "27": 1684.0, - "28": 1866.0, - "29": 1939.0, - "30": 1970.0, - "31": 1940.0, - "32": 1908.0, - "33": 1954.0, - "34": 2074.0, - "35": 2035.0, - "36": 1969.0, - "37": 2171.0, - "38": 2085.0, - "39": 2307.0, - "40": 2354.0, - "41": 2281.0, - "42": 1999.0, - "43": 2430.0, - "44": 2201.0, - "45": 2656.0, - "46": 2281.0, - "47": 2477.0, - "48": 2552.0, - "49": 2811.0, - "50": 2570.0, - "51": 2414.0, - "52": 2753.0, - "53": 2603.0, - "54": 2894.0, - "55": 2730.0, - "56": 2607.0, - "57": 2214.0, - "58": 3601.0, - "59": 2895.0, - "60": 2853.0, - "61": 2642.0, - "62": 3082.0, - "63": 3315.0, - "64": 3551.0, - "65": 2620.0, - "66": 3064.0, - "67": 3946.0, - "68": 3280.0, - "69": 2902.0, - "70": 3285.0, - "71": 3061.0, - "72": 2994.0, - "73": 3480.0, - "74": 3147.0, - "75": 3252.0, - "76": 3151.0, - "77": 3721.0, - "78": 3131.0, - "79": 3305.0, - "80": 3197.0, - "81": 3396.0, - "82": 3230.0, - "83": 3266.0, - "84": 2830.0, - "85": 2720.0, - "86": 3098.0, - "87": 2969.0, - "88": 3044.0, - "89": 3029.0, - "90": 3586.0, - "91": 3189.0, - "92": 2969.0, - "93": 2896.0, - "94": 3171.0, - "95": 3403.0, - "96": 3597.0, - "97": 3484.0, - "98": 3353.0, - "99": 3320.0, - "100": 3368.0 + "8": 1761.0, + "9": 1829.0, + "10": 1835.0, + "11": 1692.0, + "12": 1719.0, + "13": 1771.0, + "14": 1925.0, + "15": 1565.0, + "16": 1676.0, + "17": 1839.0, + "18": 1831.0, + "19": 1657.0, + "20": 1646.0, + "21": 1758.0, + "22": 1764.0, + "23": 1679.0, + "24": 1796.0, + "25": 1803.0, + "26": 1861.0, + "27": 1680.0, + "28": 1836.0, + "29": 1974.0, + "30": 1870.0, + "31": 1991.0, + "32": 1855.0, + "33": 1929.0, + "34": 1979.0, + "35": 2026.0, + "36": 1997.0, + "37": 2154.0, + "38": 2182.0, + "39": 2162.0, + "40": 2249.0, + "41": 2357.0, + "42": 1947.0, + "43": 2334.0, + "44": 2145.0, + "45": 2601.0, + "46": 2416.0, + "47": 2528.0, + "48": 2505.0, + "49": 2807.0, + "50": 2606.0, + "51": 2529.0, + "52": 2706.0, + "53": 2546.0, + "54": 2820.0, + "55": 2723.0, + "56": 2570.0, + "57": 2215.0, + "58": 3573.0, + "59": 2904.0, + "60": 2916.0, + "61": 2765.0, + "62": 3181.0, + "63": 3220.0, + "64": 3594.0, + "65": 2595.0, + "66": 3023.0, + "67": 3840.0, + "68": 3470.0, + "69": 2927.0, + "70": 3210.0, + "71": 3080.0, + "72": 2938.0, + "73": 3455.0, + "74": 3146.0, + "75": 3283.0, + "76": 3170.0, + "77": 3808.0, + "78": 3264.0, + "79": 3266.0, + "80": 3117.0, + "81": 3423.0, + "82": 3087.0, + "83": 3214.0, + "84": 2912.0, + "85": 2742.0, + "86": 3145.0, + "87": 2981.0, + "88": 3157.0, + "89": 2922.0, + "90": 3525.0, + "91": 3112.0, + "92": 3039.0, + "93": 3001.0, + "94": 3145.0, + "95": 3363.0, + "96": 3581.0, + "97": 3603.0, + "98": 3403.0, + "99": 3370.0, + "100": 3303.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.42058, - "3": 2.9906, - "4": 2.28433, - "5": 2.31945, - "6": 2.25606, - "7": 2.65053, - "8": 2.78894, - "9": 3.17745, - "10": 2.20994, - "11": 2.7953, - "12": 2.04648, - "13": 2.62183, - "14": 2.47023, - "15": 1.89176, - "16": 2.39045, - "17": 2.96348, - "18": 2.17915, - "19": 2.73072, - "20": 2.39913, - "21": 2.50469, - "22": 2.02961, - "23": 2.85935, - "24": 2.42236, - "25": 2.32274, - "26": 2.22901, - "27": 2.10895, - "28": 2.0858, - "29": 1.81038, - "30": 2.00374, - "31": 1.79224, - "32": 2.98457, - "33": 1.94536, - "34": 1.56607, - "35": 2.24687, - "36": 2.6891, - "37": 2.1153, - "38": 1.78402, - "39": 2.49259, - "40": 1.97637, - "41": 2.38065, - "42": 2.98564, - "43": 2.14245, - "44": 2.15079, - "45": 2.79879, - "46": 2.57605, - "47": 2.80077, - "48": 3.21632, - "49": 2.19134, - "50": 2.51729, - "51": 1.92835, - "52": 2.61796, - "53": 2.33409, - "54": 2.21492, - "55": 1.80381, - "56": 2.84132, - "57": 1.31239, - "58": 2.09837, - "59": 2.19106, - "60": 1.96286, - "61": 2.73324, - "62": 2.32317, - "63": 2.02322, - "64": 2.3249, - "65": 2.3802, - "66": 2.53532, - "67": 2.51893, - "68": 1.93902, - "69": 2.55482, - "70": 2.30937, - "71": 3.43239, - "72": 2.34936, - "73": 2.13696, - "74": 2.48062, - "75": 2.74573, - "76": 2.41693, - "77": 2.97019, - "78": 2.30417, - "79": 2.38768, - "80": 2.36351, - "81": 2.57248, - "82": 2.47399, - "83": 2.40451, - "84": 1.97887, - "85": 2.05544, - "86": 2.1894, - "87": 2.6966, - "88": 2.3178, - "89": 2.48366, - "90": 2.02263, - "91": 2.45368, - "92": 2.61381, - "93": 1.61994, - "94": 2.41232, - "95": 2.31977, - "96": 2.39685, - "97": 1.61519, - "98": 2.05838, - "99": 2.51131, - "100": 1.98917 + "2": 5.74561, + "3": 2.32939, + "4": 2.03341, + "5": 1.45059, + "6": 2.10676, + "7": 1.82349, + "8": 2.36974, + "9": 2.38045, + "10": 1.95152, + "11": 1.70488, + "12": 2.10831, + "13": 2.21898, + "14": 2.43806, + "15": 2.11163, + "16": 1.88284, + "17": 2.44606, + "18": 1.61767, + "19": 1.68242, + "20": 1.93656, + "21": 2.29813, + "22": 1.51289, + "23": 1.93031, + "24": 1.47823, + "25": 2.14335, + "26": 2.44124, + "27": 1.84333, + "28": 1.83233, + "29": 1.26986, + "30": 1.70074, + "31": 1.43801, + "32": 2.51519, + "33": 1.2394, + "34": 1.53585, + "35": 2.21517, + "36": 1.89411, + "37": 1.72922, + "38": 1.43561, + "39": 2.1414, + "40": 1.96952, + "41": 1.47645, + "42": 1.66898, + "43": 1.44406, + "44": 1.41459, + "45": 1.83712, + "46": 1.80574, + "47": 1.33609, + "48": 1.82871, + "49": 1.40596, + "50": 1.89905, + "51": 0.93968, + "52": 1.82141, + "53": 2.02394, + "54": 1.95526, + "55": 1.92482, + "56": 2.04566, + "57": 1.0051, + "58": 1.52745, + "59": 1.3949, + "60": 1.45945, + "61": 1.69824, + "62": 2.00334, + "63": 1.42759, + "64": 1.57136, + "65": 1.46733, + "66": 2.07538, + "67": 1.61465, + "68": 1.39343, + "69": 1.7597, + "70": 1.63956, + "71": 1.86881, + "72": 1.73014, + "73": 1.7167, + "74": 1.29422, + "75": 1.27148, + "76": 1.58482, + "77": 1.83773, + "78": 1.51511, + "79": 1.83122, + "80": 2.06212, + "81": 2.068, + "82": 2.18036, + "83": 1.44352, + "84": 1.33204, + "85": 1.40154, + "86": 1.39353, + "87": 1.98656, + "88": 1.9426, + "89": 2.18739, + "90": 1.93024, + "91": 2.0207, + "92": 1.95491, + "93": 1.41942, + "94": 1.23204, + "95": 1.47893, + "96": 1.46235, + "97": 1.21448, + "98": 1.50022, + "99": 2.30373, + "100": 1.69655 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json index 273b86af0eb..4fe660991ba 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92706, - "2": 10.92279, - "3": 10.93187, - "4": 10.93517, - "5": 10.93088, - "6": 10.92314, - "7": 10.935, - "8": 10.92462, - "9": 10.92994, - "10": 10.92922, - "11": 10.91776, - "12": 10.92571, - "13": 10.91876, + "1": 10.92698, + "2": 10.9227, + "3": 10.9319, + "4": 10.93526, + "5": 10.93085, + "6": 10.92306, + "7": 10.93505, + "8": 10.92461, + "9": 10.92991, + "10": 10.92925, + "11": 10.91773, + "12": 10.92574, + "13": 10.91873, "14": 10.90866, - "15": 10.88351, - "16": 10.87865, - "17": 10.88879, - "18": 10.8647, - "19": 10.8723, - "20": 10.78245, - "21": 10.78372, - "22": 10.76888, - "23": 10.7643, - "24": 10.7292, - "25": 10.73568, - "26": 10.71326, - "27": 10.68182, - "28": 10.60192, - "29": 10.58355, - "30": 10.55748, - "31": 10.56241, - "32": 10.53489, + "15": 10.88358, + "16": 10.87866, + "17": 10.88886, + "18": 10.86471, + "19": 10.87233, + "20": 10.78255, + "21": 10.78375, + "22": 10.769, + "23": 10.76442, + "24": 10.72927, + "25": 10.73567, + "26": 10.71336, + "27": 10.6818, + "28": 10.60193, + "29": 10.58356, + "30": 10.55753, + "31": 10.5625, + "32": 10.5349, "33": 10.50278, - "34": 10.46238, - "35": 10.47709, - "36": 10.4569, - "37": 10.41867, - "38": 10.41656, - "39": 10.38139, - "40": 10.37436, - "41": 10.3427, - "42": 10.33072, - "43": 10.3091, - "44": 10.27515, - "45": 10.30082, - "46": 10.25975, - "47": 10.2376, - "48": 10.19352, - "49": 10.18477, - "50": 10.1973, - "51": 10.19463, - "52": 10.14642, - "53": 10.15199, - "54": 10.11839, - "55": 10.0834, - "56": 10.12273, - "57": 10.10527, - "58": 10.12069, - "59": 10.06308, - "60": 10.08102, - "61": 10.0388, - "62": 10.00454, - "63": 10.07807, - "64": 10.03218, - "65": 10.0031, - "66": 10.03544, - "67": 10.00622, - "68": 9.96933, - "69": 9.99099, - "70": 9.97868, - "71": 9.99799, - "72": 9.97581, - "73": 9.9659, - "74": 9.95983, - "75": 9.92572, - "76": 9.96561, - "77": 9.95586, - "78": 9.90228, - "79": 9.91035, - "80": 9.92617, - "81": 9.94443, - "82": 9.88905, - "83": 9.84927, - "84": 9.78863, - "85": 9.77638, - "86": 9.8798, - "87": 9.91274, - "88": 9.88586, - "89": 9.82377, - "90": 9.81124, - "91": 9.82082, - "92": 9.81006, - "93": 9.75214, - "94": 9.82524, - "95": 9.81926, - "96": 9.80583, - "97": 9.74352, - "98": 9.77347, - "99": 9.8164, - "100": 9.71043 + "34": 10.46248, + "35": 10.47715, + "36": 10.45697, + "37": 10.4187, + "38": 10.41664, + "39": 10.38143, + "40": 10.37437, + "41": 10.34276, + "42": 10.33077, + "43": 10.30916, + "44": 10.2752, + "45": 10.30087, + "46": 10.25977, + "47": 10.23764, + "48": 10.19356, + "49": 10.18482, + "50": 10.19738, + "51": 10.1947, + "52": 10.14645, + "53": 10.15204, + "54": 10.11846, + "55": 10.08345, + "56": 10.12277, + "57": 10.1054, + "58": 10.12074, + "59": 10.06313, + "60": 10.08108, + "61": 10.03883, + "62": 10.00458, + "63": 10.07813, + "64": 10.03224, + "65": 10.00315, + "66": 10.03547, + "67": 10.00629, + "68": 9.96938, + "69": 9.99101, + "70": 9.97878, + "71": 9.99803, + "72": 9.97585, + "73": 9.96598, + "74": 9.95985, + "75": 9.9258, + "76": 9.96563, + "77": 9.95588, + "78": 9.90231, + "79": 9.91042, + "80": 9.9262, + "81": 9.9445, + "82": 9.88904, + "83": 9.84931, + "84": 9.78866, + "85": 9.7764, + "86": 9.87986, + "87": 9.91273, + "88": 9.88591, + "89": 9.8238, + "90": 9.81128, + "91": 9.82089, + "92": 9.81009, + "93": 9.75217, + "94": 9.82529, + "95": 9.81925, + "96": 9.80588, + "97": 9.7436, + "98": 9.77346, + "99": 9.81647, + "100": 9.71052 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1734.0, - "2": 1786.0, - "3": 1803.0, - "4": 1688.0, - "5": 1754.0, - "6": 1745.0, - "7": 1999.0, - "8": 1746.0, - "9": 1851.0, - "10": 1758.0, - "11": 1707.0, - "12": 1670.0, - "13": 1808.0, - "14": 1971.0, - "15": 1576.0, - "16": 1798.0, - "17": 1775.0, - "18": 1779.0, - "19": 1735.0, - "20": 1721.0, - "21": 1737.0, - "22": 1741.0, - "23": 1725.0, - "24": 1771.0, - "25": 1679.0, - "26": 1785.0, - "27": 1814.0, - "28": 1773.0, - "29": 1950.0, - "30": 1873.0, - "31": 1960.0, - "32": 1922.0, - "33": 1784.0, - "34": 2086.0, - "35": 1969.0, - "36": 2017.0, - "37": 2126.0, - "38": 2057.0, - "39": 2201.0, - "40": 2326.0, - "41": 2430.0, - "42": 2017.0, - "43": 2418.0, - "44": 2153.0, - "45": 2578.0, - "46": 2194.0, - "47": 2454.0, - "48": 2644.0, - "49": 2878.0, - "50": 2585.0, - "51": 2589.0, - "52": 2772.0, - "53": 2620.0, - "54": 2882.0, - "55": 2562.0, - "56": 2638.0, - "57": 2236.0, - "58": 3618.0, - "59": 2907.0, - "60": 3104.0, - "61": 2695.0, + "1": 1760.0, + "2": 1718.0, + "3": 1911.0, + "4": 1680.0, + "5": 1724.0, + "6": 1726.0, + "7": 1938.0, + "8": 1714.0, + "9": 1777.0, + "10": 1782.0, + "11": 1731.0, + "12": 1732.0, + "13": 1901.0, + "14": 1879.0, + "15": 1638.0, + "16": 1777.0, + "17": 1870.0, + "18": 1819.0, + "19": 1636.0, + "20": 1783.0, + "21": 1782.0, + "22": 1829.0, + "23": 1680.0, + "24": 1788.0, + "25": 1765.0, + "26": 1787.0, + "27": 1775.0, + "28": 1761.0, + "29": 1969.0, + "30": 1913.0, + "31": 2024.0, + "32": 1965.0, + "33": 1939.0, + "34": 2098.0, + "35": 2015.0, + "36": 1999.0, + "37": 2184.0, + "38": 2127.0, + "39": 2269.0, + "40": 2213.0, + "41": 2434.0, + "42": 2011.0, + "43": 2385.0, + "44": 2183.0, + "45": 2492.0, + "46": 2353.0, + "47": 2532.0, + "48": 2550.0, + "49": 2889.0, + "50": 2573.0, + "51": 2495.0, + "52": 2757.0, + "53": 2559.0, + "54": 2871.0, + "55": 2674.0, + "56": 2624.0, + "57": 2119.0, + "58": 3445.0, + "59": 2871.0, + "60": 3009.0, + "61": 2660.0, "62": 3290.0, - "63": 3374.0, - "64": 3512.0, - "65": 2598.0, - "66": 3025.0, - "67": 3825.0, - "68": 3285.0, - "69": 2935.0, - "70": 3254.0, - "71": 3251.0, - "72": 2977.0, - "73": 3407.0, - "74": 3152.0, - "75": 3256.0, - "76": 3120.0, - "77": 3711.0, - "78": 3219.0, - "79": 3165.0, - "80": 3018.0, - "81": 3312.0, - "82": 3215.0, - "83": 3166.0, - "84": 2938.0, - "85": 2700.0, - "86": 3277.0, - "87": 2976.0, - "88": 3161.0, - "89": 2993.0, - "90": 3642.0, - "91": 3049.0, - "92": 2919.0, - "93": 3041.0, - "94": 3165.0, - "95": 3290.0, - "96": 3514.0, - "97": 3669.0, - "98": 3398.0, - "99": 3337.0, - "100": 3183.0 + "63": 3225.0, + "64": 3763.0, + "65": 2612.0, + "66": 3044.0, + "67": 3757.0, + "68": 3424.0, + "69": 2882.0, + "70": 3216.0, + "71": 3121.0, + "72": 3021.0, + "73": 3434.0, + "74": 3146.0, + "75": 3203.0, + "76": 3066.0, + "77": 3704.0, + "78": 3253.0, + "79": 3121.0, + "80": 3113.0, + "81": 3417.0, + "82": 3220.0, + "83": 3187.0, + "84": 2939.0, + "85": 2734.0, + "86": 3071.0, + "87": 3029.0, + "88": 3078.0, + "89": 3030.0, + "90": 3639.0, + "91": 3109.0, + "92": 2912.0, + "93": 3082.0, + "94": 3308.0, + "95": 3517.0, + "96": 3691.0, + "97": 3679.0, + "98": 3363.0, + "99": 3332.0, + "100": 3322.0 } }, "mem-allocated-bytes": { @@ -325,7 +325,7 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1939785728.0, + "1": 1938737152.0, "2": 2222434304.0, "3": 2222434304.0, "4": 2222434304.0, @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.80433, - "3": 0.14746, - "4": 0.1326, - "5": 0.13168, - "6": 0.12972, - "7": 0.12874, - "8": 0.12924, - "9": 0.13018, - "10": 0.13155, - "11": 0.13133, - "12": 0.12954, - "13": 0.13049, - "14": 0.131, - "15": 0.13125, - "16": 0.13119, - "17": 0.13008, - "18": 0.13065, - "19": 0.13078, - "20": 0.13105, - "21": 0.13267, - "22": 0.13265, - "23": 0.13085, - "24": 0.13013, - "25": 0.13089, - "26": 0.13049, - "27": 0.13208, - "28": 0.13032, - "29": 0.13068, - "30": 0.13197, - "31": 0.13004, - "32": 0.13077, - "33": 0.13094, - "34": 0.13123, - "35": 0.13176, - "36": 0.13245, - "37": 0.13264, - "38": 0.13254, - "39": 0.13234, - "40": 0.13175, - "41": 0.13035, - "42": 0.13202, - "43": 0.13205, - "44": 0.13203, - "45": 0.13099, - "46": 0.13073, - "47": 0.13025, - "48": 0.13218, - "49": 0.13106, - "50": 0.1337, - "51": 0.32096, - "52": 0.13955, - "53": 0.15916, - "54": 0.13344, - "55": 0.1319, - "56": 0.13175, - "57": 0.1309, - "58": 0.13078, - "59": 0.13143, - "60": 0.13246, - "61": 0.1305, - "62": 0.1325, - "63": 0.13258, - "64": 0.13016, - "65": 0.13152, - "66": 0.13299, - "67": 0.13225, - "68": 0.1329, - "69": 0.1306, - "70": 0.13267, - "71": 0.13223, - "72": 0.13153, - "73": 0.13079, - "74": 0.13162, - "75": 0.13178, - "76": 0.13193, - "77": 0.13129, - "78": 0.13058, - "79": 0.13047, - "80": 0.13022, - "81": 0.13233, - "82": 0.13191, - "83": 0.13065, - "84": 0.13116, - "85": 0.13192, - "86": 0.13283, - "87": 0.13557, - "88": 0.13206, - "89": 0.13166, - "90": 0.13306, - "91": 0.13158, - "92": 0.13128, - "93": 0.13056, - "94": 0.1305, - "95": 0.1307, - "96": 0.13119, - "97": 0.13011, - "98": 0.13035, - "99": 0.13084, - "100": 0.13182 + "2": 7.68939, + "3": 0.16452, + "4": 0.2573, + "5": 0.10701, + "6": 0.10902, + "7": 0.1083, + "8": 0.10745, + "9": 0.107, + "10": 0.10742, + "11": 0.10605, + "12": 0.10748, + "13": 0.11109, + "14": 0.11092, + "15": 0.10938, + "16": 0.10672, + "17": 0.10684, + "18": 0.10842, + "19": 0.10563, + "20": 0.10687, + "21": 0.10683, + "22": 0.10719, + "23": 0.1077, + "24": 0.10798, + "25": 0.10923, + "26": 0.10776, + "27": 0.10724, + "28": 0.1066, + "29": 0.10863, + "30": 0.10919, + "31": 0.10829, + "32": 0.10648, + "33": 0.10778, + "34": 0.10976, + "35": 0.10789, + "36": 0.10914, + "37": 0.11071, + "38": 0.10988, + "39": 0.10941, + "40": 0.10734, + "41": 0.10622, + "42": 0.10657, + "43": 0.10846, + "44": 0.10758, + "45": 0.11097, + "46": 0.10957, + "47": 0.10845, + "48": 0.10809, + "49": 0.10826, + "50": 0.11048, + "51": 0.21285, + "52": 0.1587, + "53": 0.1183, + "54": 0.11007, + "55": 0.10899, + "56": 0.10944, + "57": 0.10877, + "58": 0.10893, + "59": 0.1105, + "60": 0.10991, + "61": 0.11037, + "62": 0.1074, + "63": 0.10993, + "64": 0.10997, + "65": 0.11052, + "66": 0.11076, + "67": 0.11019, + "68": 0.1118, + "69": 0.1106, + "70": 0.11002, + "71": 0.11053, + "72": 0.10929, + "73": 0.14614, + "74": 0.11104, + "75": 0.11271, + "76": 0.10781, + "77": 0.10581, + "78": 0.1093, + "79": 0.11008, + "80": 0.10938, + "81": 0.10953, + "82": 0.10735, + "83": 0.10955, + "84": 0.10686, + "85": 0.10852, + "86": 0.10939, + "87": 0.10792, + "88": 0.1105, + "89": 0.11322, + "90": 0.11328, + "91": 0.11112, + "92": 0.11092, + "93": 0.11237, + "94": 0.11833, + "95": 0.11106, + "96": 0.11165, + "97": 0.11234, + "98": 0.11197, + "99": 0.11138, + "100": 0.11449 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json index 44a40c1ca1b..eca2ac2f72d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95939, - "2": 10.94561, - "3": 10.94918, + "1": 10.95945, + "2": 10.94558, + "3": 10.94922, "4": 10.94704, - "5": 10.9559, - "6": 10.95226, - "7": 10.95018, - "8": 10.95679, - "9": 10.94257, - "10": 10.94725, - "11": 10.9355, + "5": 10.95589, + "6": 10.95218, + "7": 10.95016, + "8": 10.95689, + "9": 10.94261, + "10": 10.94724, + "11": 10.93546, "12": 10.94741, - "13": 10.9199, - "14": 10.92082, - "15": 10.90466, - "16": 10.89239, - "17": 10.89288, - "18": 10.89246, - "19": 10.87971, - "20": 10.80737, - "21": 10.78841, - "22": 10.77824, - "23": 10.78255, - "24": 10.75943, - "25": 10.75771, - "26": 10.73464, - "27": 10.69864, - "28": 10.61058, - "29": 10.57945, - "30": 10.56536, - "31": 10.56477, - "32": 10.55151, - "33": 10.52212, - "34": 10.47131, - "35": 10.48348, - "36": 10.46229, - "37": 10.42487, - "38": 10.42072, - "39": 10.39099, - "40": 10.37297, - "41": 10.35274, - "42": 10.35024, - "43": 10.31689, - "44": 10.30513, - "45": 10.2969, - "46": 10.26609, - "47": 10.25516, - "48": 10.21044, - "49": 10.20228, - "50": 10.21713, - "51": 10.21029, - "52": 10.15829, - "53": 10.15994, - "54": 10.13462, - "55": 10.11077, - "56": 10.12861, - "57": 10.11008, - "58": 10.12175, - "59": 10.06878, - "60": 10.08995, - "61": 10.04525, - "62": 10.01558, - "63": 10.08025, - "64": 10.03236, - "65": 10.00171, - "66": 10.04585, - "67": 10.01598, - "68": 9.98431, - "69": 9.99678, - "70": 9.98004, - "71": 10.00125, - "72": 9.98844, + "13": 10.91988, + "14": 10.92077, + "15": 10.90467, + "16": 10.8924, + "17": 10.89279, + "18": 10.89248, + "19": 10.87973, + "20": 10.80736, + "21": 10.78846, + "22": 10.77833, + "23": 10.78262, + "24": 10.75945, + "25": 10.75778, + "26": 10.73468, + "27": 10.69867, + "28": 10.61066, + "29": 10.57956, + "30": 10.56539, + "31": 10.56486, + "32": 10.55161, + "33": 10.52219, + "34": 10.47136, + "35": 10.48355, + "36": 10.46232, + "37": 10.42496, + "38": 10.42078, + "39": 10.39103, + "40": 10.37295, + "41": 10.35282, + "42": 10.35034, + "43": 10.31696, + "44": 10.30514, + "45": 10.29693, + "46": 10.26616, + "47": 10.2552, + "48": 10.21043, + "49": 10.2023, + "50": 10.21715, + "51": 10.21037, + "52": 10.1583, + "53": 10.15998, + "54": 10.13463, + "55": 10.11084, + "56": 10.12869, + "57": 10.11006, + "58": 10.12183, + "59": 10.06883, + "60": 10.09, + "61": 10.0453, + "62": 10.01564, + "63": 10.08031, + "64": 10.03238, + "65": 10.00175, + "66": 10.04593, + "67": 10.01595, + "68": 9.98437, + "69": 9.99683, + "70": 9.98003, + "71": 10.00129, + "72": 9.98841, "73": 9.97265, - "74": 9.96397, - "75": 9.92908, - "76": 9.95782, - "77": 9.95865, - "78": 9.91024, - "79": 9.9085, - "80": 9.92434, - "81": 9.94839, - "82": 9.8887, - "83": 9.85304, - "84": 9.79711, - "85": 9.7777, - "86": 9.88374, - "87": 9.90613, - "88": 9.87919, - "89": 9.81825, - "90": 9.81548, - "91": 9.81755, - "92": 9.81598, - "93": 9.74136, - "94": 9.82061, - "95": 9.80831, - "96": 9.79427, - "97": 9.73959, - "98": 9.76307, - "99": 9.81733, - "100": 9.70258 + "74": 9.96398, + "75": 9.92912, + "76": 9.95785, + "77": 9.95872, + "78": 9.91029, + "79": 9.90855, + "80": 9.92441, + "81": 9.94842, + "82": 9.88868, + "83": 9.85308, + "84": 9.79712, + "85": 9.77773, + "86": 9.88383, + "87": 9.90616, + "88": 9.87921, + "89": 9.8183, + "90": 9.81557, + "91": 9.81758, + "92": 9.81601, + "93": 9.74139, + "94": 9.82066, + "95": 9.80836, + "96": 9.79432, + "97": 9.73962, + "98": 9.76311, + "99": 9.81735, + "100": 9.70263 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1688.0, - "2": 1724.0, - "3": 1783.0, - "4": 1601.0, - "5": 1744.0, - "6": 1688.0, - "7": 1868.0, - "8": 1702.0, - "9": 1745.0, - "10": 1683.0, - "11": 1563.0, - "12": 1636.0, - "13": 1778.0, - "14": 1773.0, - "15": 1694.0, + "1": 1757.0, + "2": 1634.0, + "3": 1778.0, + "4": 1689.0, + "5": 1718.0, + "6": 1665.0, + "7": 1932.0, + "8": 1688.0, + "9": 1691.0, + "10": 1675.0, + "11": 1611.0, + "12": 1641.0, + "13": 1752.0, + "14": 1772.0, + "15": 1704.0, "16": 1707.0, - "17": 1779.0, - "18": 1679.0, - "19": 1620.0, - "20": 1646.0, - "21": 1760.0, - "22": 1708.0, - "23": 1621.0, - "24": 1776.0, - "25": 1673.0, - "26": 1769.0, - "27": 1866.0, - "28": 1756.0, - "29": 1903.0, - "30": 1877.0, - "31": 2055.0, - "32": 2002.0, - "33": 2047.0, - "34": 1991.0, - "35": 2000.0, - "36": 1885.0, - "37": 2216.0, - "38": 2158.0, - "39": 2212.0, - "40": 2320.0, - "41": 2350.0, - "42": 2074.0, - "43": 2261.0, - "44": 2201.0, - "45": 2469.0, - "46": 2322.0, - "47": 2456.0, - "48": 2498.0, - "49": 2759.0, - "50": 2514.0, - "51": 2446.0, - "52": 2598.0, - "53": 2592.0, - "54": 2706.0, - "55": 2471.0, - "56": 2631.0, - "57": 2252.0, - "58": 3577.0, - "59": 2890.0, - "60": 2988.0, - "61": 2621.0, - "62": 3002.0, - "63": 3226.0, - "64": 3493.0, - "65": 2609.0, - "66": 2957.0, - "67": 3658.0, - "68": 3215.0, - "69": 2894.0, - "70": 3265.0, - "71": 3065.0, - "72": 2937.0, - "73": 3491.0, - "74": 3258.0, - "75": 2911.0, - "76": 3278.0, - "77": 3785.0, - "78": 3230.0, - "79": 3147.0, - "80": 3018.0, - "81": 3468.0, - "82": 3034.0, - "83": 3111.0, - "84": 2921.0, - "85": 2608.0, - "86": 2990.0, - "87": 2705.0, - "88": 3045.0, - "89": 3038.0, - "90": 3834.0, - "91": 3046.0, - "92": 3063.0, - "93": 2974.0, - "94": 3253.0, - "95": 3215.0, - "96": 3509.0, - "97": 3520.0, - "98": 3331.0, - "99": 3062.0, - "100": 2906.0 + "17": 1774.0, + "18": 1653.0, + "19": 1654.0, + "20": 1674.0, + "21": 1884.0, + "22": 1717.0, + "23": 1682.0, + "24": 1842.0, + "25": 1672.0, + "26": 1827.0, + "27": 1781.0, + "28": 1787.0, + "29": 1874.0, + "30": 1756.0, + "31": 2047.0, + "32": 2000.0, + "33": 2016.0, + "34": 1927.0, + "35": 2059.0, + "36": 1953.0, + "37": 2219.0, + "38": 2079.0, + "39": 2287.0, + "40": 2267.0, + "41": 2353.0, + "42": 1984.0, + "43": 2366.0, + "44": 2237.0, + "45": 2406.0, + "46": 2437.0, + "47": 2490.0, + "48": 2610.0, + "49": 2824.0, + "50": 2462.0, + "51": 2550.0, + "52": 2650.0, + "53": 2604.0, + "54": 2782.0, + "55": 2379.0, + "56": 2607.0, + "57": 2254.0, + "58": 3543.0, + "59": 2766.0, + "60": 2903.0, + "61": 2703.0, + "62": 3189.0, + "63": 3372.0, + "64": 3554.0, + "65": 2665.0, + "66": 2967.0, + "67": 3732.0, + "68": 3289.0, + "69": 2882.0, + "70": 3230.0, + "71": 3124.0, + "72": 2888.0, + "73": 3497.0, + "74": 3070.0, + "75": 2934.0, + "76": 3294.0, + "77": 3651.0, + "78": 3172.0, + "79": 3231.0, + "80": 3072.0, + "81": 3623.0, + "82": 3038.0, + "83": 3112.0, + "84": 2955.0, + "85": 2647.0, + "86": 2960.0, + "87": 2632.0, + "88": 2986.0, + "89": 3002.0, + "90": 3706.0, + "91": 2999.0, + "92": 2941.0, + "93": 2999.0, + "94": 3127.0, + "95": 3168.0, + "96": 3607.0, + "97": 3401.0, + "98": 3405.0, + "99": 2933.0, + "100": 3025.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 746194432.0, - "2": 746194432.0, - "3": 746194432.0, - "4": 746194432.0, - "5": 746194432.0, - "6": 746194432.0, - "7": 746194432.0, - "8": 746194432.0, - "9": 746194432.0, - "10": 746194432.0, - "11": 746194432.0, - "12": 746194432.0, - "13": 746194432.0, - "14": 746194432.0, - "15": 746194432.0, - "16": 746194432.0, - "17": 746194432.0, - "18": 746194432.0, - "19": 746194432.0, - "20": 746194432.0, - "21": 746194432.0, - "22": 746194432.0, - "23": 746194432.0, - "24": 746194432.0, - "25": 746194432.0, - "26": 746194432.0, - "27": 746194432.0, - "28": 746194432.0, - "29": 746194432.0, - "30": 746194432.0, - "31": 746194432.0, - "32": 746194432.0, - "33": 746194432.0, - "34": 746194432.0, - "35": 746194432.0, - "36": 746194432.0, - "37": 746194432.0, - "38": 746194432.0, - "39": 746194432.0, - "40": 746194432.0, - "41": 746194432.0, - "42": 746194432.0, - "43": 746194432.0, - "44": 746194432.0, - "45": 746194432.0, - "46": 746194432.0, - "47": 746194432.0, - "48": 746194432.0, - "49": 746194432.0, - "50": 746194432.0, - "51": 746194432.0, - "52": 746194432.0, - "53": 746194432.0, - "54": 746194432.0, - "55": 746194432.0, - "56": 746194432.0, - "57": 746194432.0, - "58": 746194432.0, - "59": 746194432.0, - "60": 746194432.0, - "61": 746194432.0, - "62": 746194432.0, - "63": 746194432.0, - "64": 746194432.0, - "65": 746194432.0, - "66": 746194432.0, - "67": 746194432.0, - "68": 746194432.0, - "69": 746194432.0, - "70": 746194432.0, - "71": 746194432.0, - "72": 746194432.0, - "73": 746194432.0, - "74": 746194432.0, - "75": 746194432.0, - "76": 746194432.0, - "77": 746194432.0, - "78": 746194432.0, - "79": 746194432.0, - "80": 746194432.0, - "81": 746194432.0, - "82": 746194432.0, - "83": 746194432.0, - "84": 746194432.0, - "85": 746194432.0, - "86": 746194432.0, - "87": 746194432.0, - "88": 746194432.0, - "89": 746194432.0, - "90": 746194432.0, - "91": 746194432.0, - "92": 746194432.0, - "93": 746194432.0, - "94": 746194432.0, - "95": 746194432.0, - "96": 746194432.0, - "97": 746194432.0, - "98": 746194432.0, - "99": 746194432.0, - "100": 746194432.0 + "1": 747244032.0, + "2": 747244032.0, + "3": 747244032.0, + "4": 747244032.0, + "5": 747244032.0, + "6": 747244032.0, + "7": 747244032.0, + "8": 747244032.0, + "9": 747244032.0, + "10": 747244032.0, + "11": 747244032.0, + "12": 747244032.0, + "13": 747244032.0, + "14": 747244032.0, + "15": 747244032.0, + "16": 747244032.0, + "17": 747244032.0, + "18": 747244032.0, + "19": 747244032.0, + "20": 747244032.0, + "21": 747244032.0, + "22": 747244032.0, + "23": 747244032.0, + "24": 747244032.0, + "25": 747244032.0, + "26": 747244032.0, + "27": 747244032.0, + "28": 747244032.0, + "29": 747244032.0, + "30": 747244032.0, + "31": 747244032.0, + "32": 747244032.0, + "33": 747244032.0, + "34": 747244032.0, + "35": 747244032.0, + "36": 747244032.0, + "37": 747244032.0, + "38": 747244032.0, + "39": 747244032.0, + "40": 747244032.0, + "41": 747244032.0, + "42": 747244032.0, + "43": 747244032.0, + "44": 747244032.0, + "45": 747244032.0, + "46": 747244032.0, + "47": 747244032.0, + "48": 747244032.0, + "49": 747244032.0, + "50": 747244032.0, + "51": 747244032.0, + "52": 747244032.0, + "53": 747244032.0, + "54": 747244032.0, + "55": 747244032.0, + "56": 747244032.0, + "57": 747244032.0, + "58": 747244032.0, + "59": 747244032.0, + "60": 747244032.0, + "61": 747244032.0, + "62": 747244032.0, + "63": 747244032.0, + "64": 747244032.0, + "65": 747244032.0, + "66": 747244032.0, + "67": 747244032.0, + "68": 747244032.0, + "69": 747244032.0, + "70": 747244032.0, + "71": 747244032.0, + "72": 747244032.0, + "73": 747244032.0, + "74": 747244032.0, + "75": 747244032.0, + "76": 747244032.0, + "77": 747244032.0, + "78": 747244032.0, + "79": 747244032.0, + "80": 747244032.0, + "81": 747244032.0, + "82": 747244032.0, + "83": 747244032.0, + "84": 747244032.0, + "85": 747244032.0, + "86": 747244032.0, + "87": 747244032.0, + "88": 747244032.0, + "89": 747244032.0, + "90": 747244032.0, + "91": 747244032.0, + "92": 747244032.0, + "93": 747244032.0, + "94": 747244032.0, + "95": 747244032.0, + "96": 747244032.0, + "97": 747244032.0, + "98": 747244032.0, + "99": 747244032.0, + "100": 747244032.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1927202816.0, - "2": 2209851392.0, - "3": 2209851392.0, - "4": 2209851392.0, - "5": 2209851392.0, - "6": 2209851392.0, - "7": 2209851392.0, - "8": 2209851392.0, - "9": 2209851392.0, - "10": 2209851392.0, - "11": 2209851392.0, - "12": 2209851392.0, - "13": 2209851392.0, - "14": 2209851392.0, - "15": 2209851392.0, - "16": 2209851392.0, - "17": 2209851392.0, - "18": 2209851392.0, - "19": 2209851392.0, - "20": 2209851392.0, - "21": 2209851392.0, - "22": 2209851392.0, - "23": 2209851392.0, - "24": 2209851392.0, - "25": 2209851392.0, - "26": 2209851392.0, - "27": 2209851392.0, - "28": 2209851392.0, - "29": 2209851392.0, - "30": 2209851392.0, - "31": 2209851392.0, - "32": 2209851392.0, - "33": 2209851392.0, - "34": 2209851392.0, - "35": 2209851392.0, - "36": 2209851392.0, - "37": 2209851392.0, - "38": 2209851392.0, - "39": 2209851392.0, - "40": 2209851392.0, - "41": 2209851392.0, - "42": 2209851392.0, - "43": 2209851392.0, - "44": 2209851392.0, - "45": 2209851392.0, - "46": 2209851392.0, - "47": 2209851392.0, - "48": 2209851392.0, - "49": 2209851392.0, - "50": 2209851392.0, - "51": 2209851392.0, - "52": 2209851392.0, - "53": 2209851392.0, - "54": 2209851392.0, - "55": 2209851392.0, - "56": 2209851392.0, - "57": 2209851392.0, - "58": 2209851392.0, - "59": 2209851392.0, - "60": 2209851392.0, - "61": 2209851392.0, - "62": 2209851392.0, - "63": 2209851392.0, - "64": 2209851392.0, - "65": 2209851392.0, - "66": 2209851392.0, - "67": 2209851392.0, - "68": 2209851392.0, - "69": 2209851392.0, - "70": 2209851392.0, - "71": 2209851392.0, - "72": 2209851392.0, - "73": 2209851392.0, - "74": 2209851392.0, - "75": 2209851392.0, - "76": 2209851392.0, - "77": 2209851392.0, - "78": 2209851392.0, - "79": 2209851392.0, - "80": 2209851392.0, - "81": 2209851392.0, - "82": 2209851392.0, - "83": 2209851392.0, - "84": 2209851392.0, - "85": 2209851392.0, - "86": 2209851392.0, - "87": 2209851392.0, - "88": 2209851392.0, - "89": 2209851392.0, - "90": 2209851392.0, - "91": 2209851392.0, - "92": 2209851392.0, - "93": 2209851392.0, - "94": 2209851392.0, - "95": 2209851392.0, - "96": 2209851392.0, - "97": 2209851392.0, - "98": 2209851392.0, - "99": 2209851392.0, - "100": 2209851392.0 + "2": 2211948544.0, + "3": 2211948544.0, + "4": 2211948544.0, + "5": 2211948544.0, + "6": 2211948544.0, + "7": 2211948544.0, + "8": 2211948544.0, + "9": 2211948544.0, + "10": 2211948544.0, + "11": 2211948544.0, + "12": 2211948544.0, + "13": 2211948544.0, + "14": 2211948544.0, + "15": 2211948544.0, + "16": 2211948544.0, + "17": 2211948544.0, + "18": 2211948544.0, + "19": 2211948544.0, + "20": 2211948544.0, + "21": 2211948544.0, + "22": 2211948544.0, + "23": 2211948544.0, + "24": 2211948544.0, + "25": 2211948544.0, + "26": 2211948544.0, + "27": 2211948544.0, + "28": 2211948544.0, + "29": 2211948544.0, + "30": 2211948544.0, + "31": 2211948544.0, + "32": 2211948544.0, + "33": 2211948544.0, + "34": 2211948544.0, + "35": 2211948544.0, + "36": 2211948544.0, + "37": 2211948544.0, + "38": 2211948544.0, + "39": 2211948544.0, + "40": 2211948544.0, + "41": 2211948544.0, + "42": 2211948544.0, + "43": 2211948544.0, + "44": 2211948544.0, + "45": 2211948544.0, + "46": 2211948544.0, + "47": 2211948544.0, + "48": 2211948544.0, + "49": 2211948544.0, + "50": 2211948544.0, + "51": 2211948544.0, + "52": 2211948544.0, + "53": 2211948544.0, + "54": 2211948544.0, + "55": 2211948544.0, + "56": 2211948544.0, + "57": 2211948544.0, + "58": 2211948544.0, + "59": 2211948544.0, + "60": 2211948544.0, + "61": 2211948544.0, + "62": 2211948544.0, + "63": 2211948544.0, + "64": 2211948544.0, + "65": 2211948544.0, + "66": 2211948544.0, + "67": 2211948544.0, + "68": 2211948544.0, + "69": 2211948544.0, + "70": 2211948544.0, + "71": 2211948544.0, + "72": 2211948544.0, + "73": 2211948544.0, + "74": 2211948544.0, + "75": 2211948544.0, + "76": 2211948544.0, + "77": 2211948544.0, + "78": 2211948544.0, + "79": 2211948544.0, + "80": 2211948544.0, + "81": 2211948544.0, + "82": 2211948544.0, + "83": 2211948544.0, + "84": 2211948544.0, + "85": 2211948544.0, + "86": 2211948544.0, + "87": 2211948544.0, + "88": 2211948544.0, + "89": 2211948544.0, + "90": 2211948544.0, + "91": 2211948544.0, + "92": 2211948544.0, + "93": 2211948544.0, + "94": 2211948544.0, + "95": 2211948544.0, + "96": 2211948544.0, + "97": 2211948544.0, + "98": 2211948544.0, + "99": 2211948544.0, + "100": 2211948544.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.23149, - "3": 0.12534, - "4": 0.12102, - "5": 0.11736, - "6": 0.12125, - "7": 0.1219, - "8": 0.11702, - "9": 0.11818, - "10": 0.1227, - "11": 0.12476, - "12": 0.12463, - "13": 0.12408, - "14": 0.12042, - "15": 0.11999, - "16": 0.11824, - "17": 0.12037, - "18": 0.11909, - "19": 0.11683, - "20": 0.1176, - "21": 0.11703, - "22": 0.11948, - "23": 0.11789, - "24": 0.12232, - "25": 0.12176, - "26": 0.11992, - "27": 0.11848, - "28": 0.12002, - "29": 0.1179, - "30": 0.11782, - "31": 0.11816, - "32": 0.11841, - "33": 0.12154, - "34": 0.12014, - "35": 0.12846, - "36": 0.12691, - "37": 0.13076, - "38": 0.13109, - "39": 0.12616, - "40": 0.12646, - "41": 0.12693, - "42": 0.12768, - "43": 0.1269, - "44": 0.12617, - "45": 0.13224, - "46": 0.127, - "47": 0.12744, - "48": 0.12686, - "49": 0.12762, - "50": 0.1283, - "51": 0.17128, - "52": 0.14728, - "53": 0.12912, - "54": 0.1264, - "55": 0.12311, - "56": 0.12165, - "57": 0.1215, - "58": 0.12102, - "59": 0.12058, - "60": 0.1197, - "61": 0.1201, - "62": 0.11994, - "63": 0.12062, - "64": 0.12502, - "65": 0.12135, - "66": 0.12049, - "67": 0.11994, - "68": 0.12112, - "69": 0.12052, - "70": 0.12295, - "71": 0.11968, - "72": 0.11878, - "73": 0.12087, - "74": 0.11869, - "75": 0.1189, - "76": 0.12024, - "77": 0.11901, - "78": 0.11968, - "79": 0.11774, - "80": 0.11722, - "81": 0.11826, - "82": 0.11777, - "83": 0.11878, - "84": 0.11803, - "85": 0.11918, - "86": 0.11883, - "87": 0.11808, - "88": 0.11756, - "89": 0.11874, - "90": 0.11921, - "91": 0.11788, - "92": 0.12112, - "93": 0.11795, - "94": 0.11765, - "95": 0.12368, - "96": 0.11779, - "97": 0.11789, - "98": 0.11784, - "99": 0.11675, - "100": 0.11781 + "2": 8.2229, + "3": 0.12486, + "4": 0.12226, + "5": 0.12375, + "6": 0.1252, + "7": 0.12198, + "8": 0.1218, + "9": 0.12186, + "10": 0.12188, + "11": 0.12292, + "12": 0.12247, + "13": 0.12365, + "14": 0.12388, + "15": 0.12346, + "16": 0.12431, + "17": 0.12314, + "18": 0.12347, + "19": 0.12506, + "20": 0.12398, + "21": 0.12538, + "22": 0.12586, + "23": 0.12274, + "24": 0.12323, + "25": 0.12311, + "26": 0.124, + "27": 0.12368, + "28": 0.12393, + "29": 0.12651, + "30": 0.1239, + "31": 0.12283, + "32": 0.12456, + "33": 0.12401, + "34": 0.12267, + "35": 0.12721, + "36": 0.12254, + "37": 0.12287, + "38": 0.12252, + "39": 0.12289, + "40": 0.12116, + "41": 0.12256, + "42": 0.12281, + "43": 0.1236, + "44": 0.12245, + "45": 0.12124, + "46": 0.12086, + "47": 0.12101, + "48": 0.12236, + "49": 0.12208, + "50": 0.1227, + "51": 0.15084, + "52": 0.14144, + "53": 0.12379, + "54": 0.12437, + "55": 0.12473, + "56": 0.12168, + "57": 0.12278, + "58": 0.12204, + "59": 0.12407, + "60": 0.12556, + "61": 0.12331, + "62": 0.12215, + "63": 0.12886, + "64": 0.12161, + "65": 0.12091, + "66": 0.12403, + "67": 0.12442, + "68": 0.12383, + "69": 0.12374, + "70": 0.12387, + "71": 0.12235, + "72": 0.12335, + "73": 0.12255, + "74": 0.12587, + "75": 0.12196, + "76": 0.1228, + "77": 0.1236, + "78": 0.12244, + "79": 0.12339, + "80": 0.12244, + "81": 0.123, + "82": 0.12207, + "83": 0.12301, + "84": 0.12207, + "85": 0.12305, + "86": 0.12291, + "87": 0.12347, + "88": 0.12135, + "89": 0.12334, + "90": 0.1236, + "91": 0.12228, + "92": 0.12362, + "93": 0.12399, + "94": 0.12352, + "95": 0.12265, + "96": 0.12187, + "97": 0.12389, + "98": 0.12437, + "99": 0.12258, + "100": 0.12298 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json index 4476edef5ff..3c0fadda88a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92753, - "2": 10.92559, + "1": 10.92752, + "2": 10.92554, "3": 10.93114, - "4": 10.93741, - "5": 10.93016, - "6": 10.92355, - "7": 10.93863, - "8": 10.92774, - "9": 10.93076, - "10": 10.92802, - "11": 10.91411, - "12": 10.92298, - "13": 10.92014, - "14": 10.90614, + "4": 10.93727, + "5": 10.93015, + "6": 10.92346, + "7": 10.93864, + "8": 10.92777, + "9": 10.93074, + "10": 10.92801, + "11": 10.91409, + "12": 10.92299, + "13": 10.92006, + "14": 10.90618, "15": 10.88338, - "16": 10.87951, - "17": 10.88868, - "18": 10.86451, - "19": 10.87355, - "20": 10.783, - "21": 10.78553, - "22": 10.76764, - "23": 10.764, - "24": 10.72663, - "25": 10.73401, - "26": 10.71265, - "27": 10.67569, - "28": 10.6012, - "29": 10.58219, - "30": 10.55702, - "31": 10.56167, - "32": 10.53487, - "33": 10.50348, - "34": 10.46525, - "35": 10.47887, - "36": 10.45601, - "37": 10.41808, - "38": 10.4167, - "39": 10.38178, - "40": 10.37615, - "41": 10.34328, - "42": 10.33161, - "43": 10.3077, - "44": 10.27713, - "45": 10.2988, - "46": 10.25974, - "47": 10.23655, - "48": 10.19378, - "49": 10.18543, - "50": 10.19728, - "51": 10.19367, - "52": 10.14697, - "53": 10.14982, - "54": 10.11733, - "55": 10.08476, - "56": 10.12202, - "57": 10.10466, - "58": 10.11902, - "59": 10.06425, - "60": 10.08087, + "16": 10.87953, + "17": 10.88861, + "18": 10.86454, + "19": 10.87347, + "20": 10.78318, + "21": 10.78551, + "22": 10.76768, + "23": 10.76394, + "24": 10.72672, + "25": 10.734, + "26": 10.71269, + "27": 10.6757, + "28": 10.60128, + "29": 10.5823, + "30": 10.55707, + "31": 10.56165, + "32": 10.53484, + "33": 10.50353, + "34": 10.46529, + "35": 10.47894, + "36": 10.45605, + "37": 10.41812, + "38": 10.41677, + "39": 10.38189, + "40": 10.37623, + "41": 10.34336, + "42": 10.33171, + "43": 10.30768, + "44": 10.27724, + "45": 10.29884, + "46": 10.2598, + "47": 10.23664, + "48": 10.1938, + "49": 10.18553, + "50": 10.19738, + "51": 10.1937, + "52": 10.14694, + "53": 10.1499, + "54": 10.11736, + "55": 10.08481, + "56": 10.12206, + "57": 10.10473, + "58": 10.11908, + "59": 10.06432, + "60": 10.08092, "61": 10.03845, - "62": 10.00507, - "63": 10.07872, - "64": 10.03202, - "65": 10.00346, - "66": 10.03491, - "67": 10.00571, - "68": 9.97032, - "69": 9.99153, - "70": 9.97866, - "71": 9.9998, - "72": 9.97531, - "73": 9.96566, - "74": 9.95983, + "62": 10.00515, + "63": 10.07878, + "64": 10.03206, + "65": 10.00349, + "66": 10.03495, + "67": 10.00575, + "68": 9.97033, + "69": 9.99157, + "70": 9.97869, + "71": 9.99986, + "72": 9.97535, + "73": 9.96568, + "74": 9.95986, "75": 9.92549, - "76": 9.96566, + "76": 9.96565, "77": 9.95606, - "78": 9.90446, - "79": 9.91127, - "80": 9.92478, - "81": 9.94432, - "82": 9.88863, - "83": 9.85093, - "84": 9.78899, - "85": 9.77531, - "86": 9.88013, + "78": 9.9045, + "79": 9.91133, + "80": 9.92487, + "81": 9.94438, + "82": 9.88871, + "83": 9.85105, + "84": 9.78905, + "85": 9.77533, + "86": 9.88023, "87": 9.91309, - "88": 9.88639, - "89": 9.82366, - "90": 9.81326, - "91": 9.82136, - "92": 9.81011, - "93": 9.75216, - "94": 9.82571, - "95": 9.81912, - "96": 9.8055, - "97": 9.7436, - "98": 9.77248, - "99": 9.81661, - "100": 9.71126 + "88": 9.88646, + "89": 9.8237, + "90": 9.81331, + "91": 9.82141, + "92": 9.81015, + "93": 9.75217, + "94": 9.82575, + "95": 9.8191, + "96": 9.80552, + "97": 9.74366, + "98": 9.7725, + "99": 9.8167, + "100": 9.71129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1770.0, - "2": 1703.0, - "3": 1799.0, - "4": 1823.0, - "5": 1737.0, - "6": 1733.0, + "1": 1845.0, + "2": 1697.0, + "3": 1692.0, + "4": 1714.0, + "5": 1786.0, + "6": 1732.0, "7": 1952.0, - "8": 1738.0, - "9": 1687.0, - "10": 1780.0, - "11": 1673.0, - "12": 1677.0, - "13": 1724.0, - "14": 1837.0, - "15": 1570.0, - "16": 1727.0, - "17": 1788.0, - "18": 1780.0, - "19": 1660.0, - "20": 1694.0, - "21": 1732.0, - "22": 1727.0, - "23": 1703.0, - "24": 1799.0, - "25": 1588.0, - "26": 1817.0, - "27": 1684.0, - "28": 1866.0, - "29": 1939.0, - "30": 1970.0, - "31": 1940.0, - "32": 1908.0, - "33": 1954.0, - "34": 2074.0, - "35": 2035.0, - "36": 1969.0, - "37": 2171.0, - "38": 2085.0, - "39": 2307.0, - "40": 2354.0, - "41": 2281.0, - "42": 1999.0, - "43": 2430.0, - "44": 2201.0, - "45": 2656.0, - "46": 2281.0, - "47": 2477.0, - "48": 2552.0, - "49": 2811.0, - "50": 2570.0, - "51": 2414.0, - "52": 2753.0, - "53": 2603.0, - "54": 2894.0, - "55": 2730.0, - "56": 2607.0, - "57": 2214.0, - "58": 3601.0, - "59": 2895.0, - "60": 2853.0, - "61": 2642.0, - "62": 3082.0, - "63": 3315.0, - "64": 3551.0, - "65": 2620.0, - "66": 3064.0, - "67": 3946.0, - "68": 3280.0, - "69": 2902.0, - "70": 3285.0, - "71": 3061.0, - "72": 2994.0, - "73": 3480.0, - "74": 3147.0, - "75": 3252.0, - "76": 3151.0, - "77": 3721.0, - "78": 3131.0, - "79": 3305.0, - "80": 3197.0, - "81": 3396.0, - "82": 3230.0, - "83": 3266.0, - "84": 2830.0, - "85": 2720.0, - "86": 3098.0, - "87": 2969.0, - "88": 3044.0, - "89": 3029.0, - "90": 3586.0, - "91": 3189.0, - "92": 2969.0, - "93": 2896.0, - "94": 3171.0, - "95": 3403.0, - "96": 3597.0, - "97": 3484.0, - "98": 3353.0, - "99": 3320.0, - "100": 3368.0 + "8": 1761.0, + "9": 1829.0, + "10": 1835.0, + "11": 1692.0, + "12": 1719.0, + "13": 1771.0, + "14": 1925.0, + "15": 1565.0, + "16": 1676.0, + "17": 1839.0, + "18": 1831.0, + "19": 1657.0, + "20": 1646.0, + "21": 1758.0, + "22": 1764.0, + "23": 1679.0, + "24": 1796.0, + "25": 1803.0, + "26": 1861.0, + "27": 1680.0, + "28": 1836.0, + "29": 1974.0, + "30": 1870.0, + "31": 1991.0, + "32": 1855.0, + "33": 1929.0, + "34": 1979.0, + "35": 2026.0, + "36": 1997.0, + "37": 2154.0, + "38": 2182.0, + "39": 2162.0, + "40": 2249.0, + "41": 2357.0, + "42": 1947.0, + "43": 2334.0, + "44": 2145.0, + "45": 2601.0, + "46": 2416.0, + "47": 2528.0, + "48": 2505.0, + "49": 2807.0, + "50": 2606.0, + "51": 2529.0, + "52": 2706.0, + "53": 2546.0, + "54": 2820.0, + "55": 2723.0, + "56": 2570.0, + "57": 2215.0, + "58": 3573.0, + "59": 2904.0, + "60": 2916.0, + "61": 2765.0, + "62": 3181.0, + "63": 3220.0, + "64": 3594.0, + "65": 2595.0, + "66": 3023.0, + "67": 3840.0, + "68": 3470.0, + "69": 2927.0, + "70": 3210.0, + "71": 3080.0, + "72": 2938.0, + "73": 3455.0, + "74": 3146.0, + "75": 3283.0, + "76": 3170.0, + "77": 3808.0, + "78": 3264.0, + "79": 3266.0, + "80": 3117.0, + "81": 3423.0, + "82": 3087.0, + "83": 3214.0, + "84": 2912.0, + "85": 2742.0, + "86": 3145.0, + "87": 2981.0, + "88": 3157.0, + "89": 2922.0, + "90": 3525.0, + "91": 3112.0, + "92": 3039.0, + "93": 3001.0, + "94": 3145.0, + "95": 3363.0, + "96": 3581.0, + "97": 3603.0, + "98": 3403.0, + "99": 3370.0, + "100": 3303.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.91781, - "3": 2.01209, - "4": 1.70703, - "5": 1.58348, - "6": 2.04178, - "7": 1.68383, - "8": 2.1805, - "9": 2.29254, - "10": 1.87808, - "11": 2.07274, - "12": 1.87306, - "13": 2.16607, - "14": 2.09193, - "15": 1.60699, - "16": 1.71658, - "17": 2.40105, - "18": 1.86098, - "19": 1.75259, - "20": 1.81546, - "21": 2.18524, - "22": 1.89571, - "23": 1.93775, - "24": 2.02166, - "25": 2.23446, - "26": 2.42493, - "27": 2.03299, - "28": 1.917, - "29": 1.68675, - "30": 1.97254, - "31": 1.5062, - "32": 2.99424, - "33": 1.6385, - "34": 1.692, - "35": 2.29559, - "36": 2.26179, - "37": 1.98836, - "38": 1.4767, - "39": 2.42606, - "40": 2.06154, - "41": 1.85116, - "42": 2.68553, - "43": 1.94445, - "44": 1.81498, - "45": 2.2976, - "46": 1.90665, - "47": 1.85997, - "48": 2.40369, - "49": 1.89519, - "50": 2.15372, - "51": 1.50287, - "52": 2.04698, - "53": 2.17394, - "54": 2.19961, - "55": 2.10794, - "56": 2.76028, - "57": 1.42194, - "58": 2.1181, - "59": 1.60096, - "60": 1.68702, - "61": 2.10997, - "62": 2.402, - "63": 1.76578, - "64": 2.04048, - "65": 1.92492, - "66": 2.09545, - "67": 2.18965, - "68": 1.41838, - "69": 2.07955, - "70": 2.01637, - "71": 2.36996, - "72": 1.86862, - "73": 1.90973, - "74": 1.92846, - "75": 1.62216, - "76": 1.87281, - "77": 1.84399, - "78": 1.69104, - "79": 2.26455, - "80": 1.95583, - "81": 2.16425, - "82": 2.33086, - "83": 2.02687, - "84": 1.78718, - "85": 2.01981, - "86": 1.77488, - "87": 2.50179, - "88": 2.17543, - "89": 2.25997, - "90": 1.99416, - "91": 2.10642, - "92": 2.65749, - "93": 1.71244, - "94": 2.00789, - "95": 2.13119, - "96": 1.93962, - "97": 1.87888, - "98": 2.17582, - "99": 2.72487, - "100": 1.95412 + "2": 5.0884, + "3": 1.63935, + "4": 1.76239, + "5": 1.12357, + "6": 1.42166, + "7": 1.5539, + "8": 1.7874, + "9": 2.0122, + "10": 1.84823, + "11": 1.92303, + "12": 1.38084, + "13": 1.55792, + "14": 1.9485, + "15": 1.4087, + "16": 1.57752, + "17": 1.86657, + "18": 1.44725, + "19": 1.8268, + "20": 1.34889, + "21": 1.65294, + "22": 1.52985, + "23": 1.72559, + "24": 1.57982, + "25": 2.14844, + "26": 2.10693, + "27": 1.59089, + "28": 1.47828, + "29": 1.39029, + "30": 1.49195, + "31": 1.51833, + "32": 2.52942, + "33": 1.43299, + "34": 1.30636, + "35": 1.62872, + "36": 1.88048, + "37": 1.59464, + "38": 1.48995, + "39": 1.89936, + "40": 1.66074, + "41": 1.71984, + "42": 2.24212, + "43": 1.69697, + "44": 1.40749, + "45": 1.76307, + "46": 1.68263, + "47": 1.86002, + "48": 1.69403, + "49": 1.52582, + "50": 1.72757, + "51": 1.90785, + "52": 2.00925, + "53": 1.7683, + "54": 2.30456, + "55": 1.75763, + "56": 2.32479, + "57": 1.42196, + "58": 2.11448, + "59": 1.61431, + "60": 1.67121, + "61": 1.76913, + "62": 2.14309, + "63": 1.54642, + "64": 1.79511, + "65": 1.62025, + "66": 1.99102, + "67": 2.46269, + "68": 1.12891, + "69": 1.7716, + "70": 1.7667, + "71": 2.34366, + "72": 1.63476, + "73": 1.66486, + "74": 1.76233, + "75": 1.67406, + "76": 1.7862, + "77": 1.70589, + "78": 1.90072, + "79": 2.14042, + "80": 1.99122, + "81": 2.1455, + "82": 2.07692, + "83": 1.90913, + "84": 1.71062, + "85": 1.60423, + "86": 1.70447, + "87": 2.40575, + "88": 1.78421, + "89": 1.99069, + "90": 1.67997, + "91": 1.7786, + "92": 2.0709, + "93": 1.52056, + "94": 1.52439, + "95": 1.56991, + "96": 1.54645, + "97": 1.13161, + "98": 1.87693, + "99": 2.57696, + "100": 1.76825 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json index 1956639b78e..8c64edcc508 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.91846, - "2": 10.91891, - "3": 10.90198, - "4": 10.9097, - "5": 10.91431, - "6": 10.90782, - "7": 10.91992, - "8": 10.9068, - "9": 10.90466, - "10": 10.89075, + "1": 10.91844, + "2": 10.91892, + "3": 10.90201, + "4": 10.9096, + "5": 10.91433, + "6": 10.90775, + "7": 10.91994, + "8": 10.90684, + "9": 10.90468, + "10": 10.89062, "11": 10.90827, - "12": 10.89872, - "13": 10.90246, - "14": 10.88297, - "15": 10.90017, - "16": 10.87246, - "17": 10.88772, - "18": 10.84563, - "19": 10.87269, - "20": 10.82137, - "21": 10.82632, - "22": 10.82087, - "23": 10.8091, - "24": 10.76352, - "25": 10.77787, - "26": 10.76039, - "27": 10.76298, - "28": 10.7083, - "29": 10.69251, - "30": 10.66554, - "31": 10.65902, - "32": 10.65052, - "33": 10.61636, - "34": 10.59526, - "35": 10.59157, - "36": 10.5646, - "37": 10.54742, - "38": 10.55559, - "39": 10.51852, - "40": 10.49345, - "41": 10.47134, - "42": 10.46605, - "43": 10.42334, - "44": 10.40811, - "45": 10.40976, - "46": 10.37231, - "47": 10.36063, - "48": 10.32128, - "49": 10.31015, - "50": 10.31076, - "51": 10.30076, - "52": 10.25594, - "53": 10.24776, - "54": 10.21612, - "55": 10.18791, - "56": 10.21227, - "57": 10.18849, - "58": 10.204, - "59": 10.1558, - "60": 10.15081, - "61": 10.12126, - "62": 10.08159, - "63": 10.14606, - "64": 10.09627, - "65": 10.06844, - "66": 10.09198, - "67": 10.06175, - "68": 10.02259, - "69": 10.0324, - "70": 10.02581, - "71": 10.04278, - "72": 10.0192, - "73": 9.99925, - "74": 9.98696, - "75": 9.95134, - "76": 9.99944, - "77": 9.99431, - "78": 9.93863, - "79": 9.9453, - "80": 9.9519, - "81": 9.97652, - "82": 9.92943, - "83": 9.8878, - "84": 9.82722, - "85": 9.81054, - "86": 9.9238, - "87": 9.93877, + "12": 10.89858, + "13": 10.90244, + "14": 10.88286, + "15": 10.90009, + "16": 10.87243, + "17": 10.88781, + "18": 10.84561, + "19": 10.87275, + "20": 10.82136, + "21": 10.82635, + "22": 10.82082, + "23": 10.80903, + "24": 10.76349, + "25": 10.77797, + "26": 10.76044, + "27": 10.76296, + "28": 10.70835, + "29": 10.6926, + "30": 10.66563, + "31": 10.65905, + "32": 10.65068, + "33": 10.61641, + "34": 10.5953, + "35": 10.59164, + "36": 10.56464, + "37": 10.54747, + "38": 10.55558, + "39": 10.51849, + "40": 10.49348, + "41": 10.47135, + "42": 10.46608, + "43": 10.42337, + "44": 10.40814, + "45": 10.40988, + "46": 10.37229, + "47": 10.36064, + "48": 10.3214, + "49": 10.31017, + "50": 10.31083, + "51": 10.30073, + "52": 10.25591, + "53": 10.24786, + "54": 10.21626, + "55": 10.188, + "56": 10.21229, + "57": 10.18853, + "58": 10.20407, + "59": 10.15584, + "60": 10.15084, + "61": 10.12121, + "62": 10.08166, + "63": 10.14607, + "64": 10.09626, + "65": 10.06846, + "66": 10.09203, + "67": 10.06184, + "68": 10.0226, + "69": 10.03243, + "70": 10.02579, + "71": 10.04282, + "72": 10.01921, + "73": 9.99924, + "74": 9.98707, + "75": 9.95143, + "76": 9.9995, + "77": 9.99441, + "78": 9.93867, + "79": 9.94538, + "80": 9.95194, + "81": 9.97653, + "82": 9.92946, + "83": 9.88784, + "84": 9.82725, + "85": 9.81058, + "86": 9.92382, + "87": 9.93881, "88": 9.91687, - "89": 9.85662, - "90": 9.85457, - "91": 9.86641, - "92": 9.85444, - "93": 9.78957, - "94": 9.86406, - "95": 9.86105, - "96": 9.84668, - "97": 9.7881, - "98": 9.82254, - "99": 9.86429, - "100": 9.76132 + "89": 9.8567, + "90": 9.85456, + "91": 9.86642, + "92": 9.85445, + "93": 9.78955, + "94": 9.86401, + "95": 9.86111, + "96": 9.8467, + "97": 9.78813, + "98": 9.82257, + "99": 9.86433, + "100": 9.76134 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2664.0, - "2": 2952.0, - "3": 2834.0, - "4": 2882.0, - "5": 2841.0, - "6": 2730.0, - "7": 3053.0, - "8": 2852.0, - "9": 2896.0, - "10": 2898.0, - "11": 3160.0, - "12": 2933.0, - "13": 3188.0, - "14": 3034.0, - "15": 2949.0, - "16": 3003.0, - "17": 3001.0, - "18": 2741.0, - "19": 2992.0, - "20": 3024.0, - "21": 2749.0, - "22": 2866.0, - "23": 2707.0, - "24": 2728.0, - "25": 2798.0, - "26": 2848.0, - "27": 2866.0, - "28": 2867.0, - "29": 2882.0, - "30": 2612.0, - "31": 2685.0, - "32": 2748.0, - "33": 2550.0, - "34": 2594.0, - "35": 2571.0, - "36": 2655.0, - "37": 2831.0, - "38": 2655.0, - "39": 2480.0, - "40": 2760.0, - "41": 2523.0, - "42": 2683.0, - "43": 2722.0, - "44": 2677.0, - "45": 2661.0, - "46": 2584.0, - "47": 2579.0, - "48": 2597.0, - "49": 2563.0, - "50": 2703.0, - "51": 2732.0, - "52": 2708.0, - "53": 2727.0, - "54": 2827.0, - "55": 2720.0, - "56": 2854.0, - "57": 2363.0, - "58": 2969.0, - "59": 2959.0, - "60": 2907.0, - "61": 2784.0, - "62": 2805.0, - "63": 3067.0, - "64": 3126.0, - "65": 2753.0, - "66": 3008.0, - "67": 3210.0, - "68": 3215.0, - "69": 2986.0, - "70": 3054.0, - "71": 3034.0, - "72": 3107.0, - "73": 3099.0, - "74": 2882.0, - "75": 2829.0, - "76": 3233.0, - "77": 3323.0, - "78": 3167.0, - "79": 3148.0, - "80": 3098.0, - "81": 3347.0, - "82": 3341.0, - "83": 3103.0, - "84": 3040.0, - "85": 2872.0, - "86": 3164.0, - "87": 3134.0, - "88": 3257.0, - "89": 3098.0, - "90": 3484.0, - "91": 2741.0, - "92": 3238.0, - "93": 2987.0, - "94": 3173.0, - "95": 3139.0, - "96": 3354.0, - "97": 3546.0, - "98": 3537.0, - "99": 3174.0, - "100": 3341.0 + "1": 2767.0, + "2": 2953.0, + "3": 2700.0, + "4": 2860.0, + "5": 2925.0, + "6": 2795.0, + "7": 3052.0, + "8": 2829.0, + "9": 2919.0, + "10": 2922.0, + "11": 3188.0, + "12": 2821.0, + "13": 3075.0, + "14": 2999.0, + "15": 2941.0, + "16": 2985.0, + "17": 2923.0, + "18": 2778.0, + "19": 3002.0, + "20": 3005.0, + "21": 2809.0, + "22": 2936.0, + "23": 2701.0, + "24": 2619.0, + "25": 2764.0, + "26": 2974.0, + "27": 2882.0, + "28": 2728.0, + "29": 2860.0, + "30": 2702.0, + "31": 2645.0, + "32": 2712.0, + "33": 2541.0, + "34": 2686.0, + "35": 2515.0, + "36": 2703.0, + "37": 2786.0, + "38": 2740.0, + "39": 2522.0, + "40": 2774.0, + "41": 2494.0, + "42": 2610.0, + "43": 2666.0, + "44": 2601.0, + "45": 2621.0, + "46": 2605.0, + "47": 2583.0, + "48": 2644.0, + "49": 2637.0, + "50": 2665.0, + "51": 2749.0, + "52": 2706.0, + "53": 2844.0, + "54": 2839.0, + "55": 2672.0, + "56": 2886.0, + "57": 2486.0, + "58": 3030.0, + "59": 2910.0, + "60": 2774.0, + "61": 2833.0, + "62": 2697.0, + "63": 3110.0, + "64": 3122.0, + "65": 2730.0, + "66": 2909.0, + "67": 3202.0, + "68": 3048.0, + "69": 2947.0, + "70": 2995.0, + "71": 2966.0, + "72": 3040.0, + "73": 3269.0, + "74": 3053.0, + "75": 2883.0, + "76": 3090.0, + "77": 3300.0, + "78": 3133.0, + "79": 3108.0, + "80": 3041.0, + "81": 3178.0, + "82": 3296.0, + "83": 3068.0, + "84": 2975.0, + "85": 2843.0, + "86": 3114.0, + "87": 3097.0, + "88": 3212.0, + "89": 3162.0, + "90": 3452.0, + "91": 2727.0, + "92": 3218.0, + "93": 3021.0, + "94": 3104.0, + "95": 3133.0, + "96": 3345.0, + "97": 3505.0, + "98": 3615.0, + "99": 3309.0, + "100": 3471.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 745732608.0, - "2": 745732608.0, - "3": 745732608.0, - "4": 745732608.0, - "5": 745732608.0, - "6": 745732608.0, - "7": 745732608.0, - "8": 745732608.0, - "9": 745732608.0, - "10": 745732608.0, - "11": 745732608.0, - "12": 745732608.0, - "13": 745732608.0, - "14": 745732608.0, - "15": 745732608.0, - "16": 745732608.0, - "17": 745732608.0, - "18": 745732608.0, - "19": 745732608.0, - "20": 745732608.0, - "21": 745732608.0, - "22": 745732608.0, - "23": 745732608.0, - "24": 745732608.0, - "25": 745732608.0, - "26": 745732608.0, - "27": 745732608.0, - "28": 745732608.0, - "29": 745732608.0, - "30": 745732608.0, - "31": 745732608.0, - "32": 745732608.0, - "33": 745732608.0, - "34": 745732608.0, - "35": 745732608.0, - "36": 745732608.0, - "37": 745732608.0, - "38": 745732608.0, - "39": 745732608.0, - "40": 745732608.0, - "41": 745732608.0, - "42": 745732608.0, - "43": 745732608.0, - "44": 745732608.0, - "45": 745732608.0, - "46": 745732608.0, - "47": 745732608.0, - "48": 745732608.0, - "49": 745732608.0, - "50": 745732608.0, - "51": 745732608.0, - "52": 745732608.0, - "53": 745732608.0, - "54": 745732608.0, - "55": 745732608.0, - "56": 745732608.0, - "57": 745732608.0, - "58": 745732608.0, - "59": 745732608.0, - "60": 745732608.0, - "61": 745732608.0, - "62": 745732608.0, - "63": 745732608.0, - "64": 745732608.0, - "65": 745732608.0, - "66": 745732608.0, - "67": 745732608.0, - "68": 745732608.0, - "69": 745732608.0, - "70": 745732608.0, - "71": 745732608.0, - "72": 745732608.0, - "73": 745732608.0, - "74": 745732608.0, - "75": 745732608.0, - "76": 745732608.0, - "77": 745732608.0, - "78": 745732608.0, - "79": 745732608.0, - "80": 745732608.0, - "81": 745732608.0, - "82": 745732608.0, - "83": 745732608.0, - "84": 745732608.0, - "85": 745732608.0, - "86": 745732608.0, - "87": 745732608.0, - "88": 745732608.0, - "89": 745732608.0, - "90": 745732608.0, - "91": 745732608.0, - "92": 745732608.0, - "93": 745732608.0, - "94": 745732608.0, - "95": 745732608.0, - "96": 745732608.0, - "97": 745732608.0, - "98": 745732608.0, - "99": 745732608.0, - "100": 745732608.0 + "1": 745077248.0, + "2": 745077248.0, + "3": 745077248.0, + "4": 745077248.0, + "5": 745077248.0, + "6": 745077248.0, + "7": 745077248.0, + "8": 745077248.0, + "9": 745077248.0, + "10": 745077248.0, + "11": 745077248.0, + "12": 745077248.0, + "13": 745077248.0, + "14": 745077248.0, + "15": 745077248.0, + "16": 745077248.0, + "17": 745077248.0, + "18": 745077248.0, + "19": 745077248.0, + "20": 745077248.0, + "21": 745077248.0, + "22": 745077248.0, + "23": 745077248.0, + "24": 745077248.0, + "25": 745077248.0, + "26": 745077248.0, + "27": 745077248.0, + "28": 745077248.0, + "29": 745077248.0, + "30": 745077248.0, + "31": 745077248.0, + "32": 745077248.0, + "33": 745077248.0, + "34": 745077248.0, + "35": 745077248.0, + "36": 745077248.0, + "37": 745077248.0, + "38": 745077248.0, + "39": 745077248.0, + "40": 745077248.0, + "41": 745077248.0, + "42": 745077248.0, + "43": 745077248.0, + "44": 745077248.0, + "45": 745077248.0, + "46": 745077248.0, + "47": 745077248.0, + "48": 745077248.0, + "49": 745077248.0, + "50": 745077248.0, + "51": 745077248.0, + "52": 745077248.0, + "53": 745077248.0, + "54": 745077248.0, + "55": 745077248.0, + "56": 745077248.0, + "57": 745077248.0, + "58": 745077248.0, + "59": 745077248.0, + "60": 745077248.0, + "61": 745077248.0, + "62": 745077248.0, + "63": 745077248.0, + "64": 745077248.0, + "65": 745077248.0, + "66": 745077248.0, + "67": 745077248.0, + "68": 745077248.0, + "69": 745077248.0, + "70": 745077248.0, + "71": 745077248.0, + "72": 745077248.0, + "73": 745077248.0, + "74": 745077248.0, + "75": 745077248.0, + "76": 745077248.0, + "77": 745077248.0, + "78": 745077248.0, + "79": 745077248.0, + "80": 745077248.0, + "81": 745077248.0, + "82": 745077248.0, + "83": 745077248.0, + "84": 745077248.0, + "85": 745077248.0, + "86": 745077248.0, + "87": 745077248.0, + "88": 745077248.0, + "89": 745077248.0, + "90": 745077248.0, + "91": 745077248.0, + "92": 745077248.0, + "93": 745077248.0, + "94": 745077248.0, + "95": 745077248.0, + "96": 745077248.0, + "97": 745077248.0, + "98": 745077248.0, + "99": 745077248.0, + "100": 745077248.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1940442112.0, - "2": 2223151104.0, - "3": 2223151104.0, - "4": 2223151104.0, - "5": 2223151104.0, - "6": 2223151104.0, - "7": 2223151104.0, - "8": 2223151104.0, - "9": 2223151104.0, - "10": 2223151104.0, - "11": 2223151104.0, - "12": 2223151104.0, - "13": 2223151104.0, - "14": 2223151104.0, - "15": 2223151104.0, - "16": 2223151104.0, - "17": 2223151104.0, - "18": 2223151104.0, - "19": 2223151104.0, - "20": 2223151104.0, - "21": 2223151104.0, - "22": 2223151104.0, - "23": 2223151104.0, - "24": 2223151104.0, - "25": 2223151104.0, - "26": 2223151104.0, - "27": 2223151104.0, - "28": 2223151104.0, - "29": 2223151104.0, - "30": 2223151104.0, - "31": 2223151104.0, - "32": 2223151104.0, - "33": 2223151104.0, - "34": 2223151104.0, - "35": 2223151104.0, - "36": 2223151104.0, - "37": 2223151104.0, - "38": 2223151104.0, - "39": 2223151104.0, - "40": 2223151104.0, - "41": 2223151104.0, - "42": 2223151104.0, - "43": 2223151104.0, - "44": 2223151104.0, - "45": 2223151104.0, - "46": 2223151104.0, - "47": 2223151104.0, - "48": 2223151104.0, - "49": 2223151104.0, - "50": 2223151104.0, - "51": 2223151104.0, - "52": 2223151104.0, - "53": 2223151104.0, - "54": 2223151104.0, - "55": 2223151104.0, - "56": 2223151104.0, - "57": 2223151104.0, - "58": 2223151104.0, - "59": 2223151104.0, - "60": 2223151104.0, - "61": 2223151104.0, - "62": 2223151104.0, - "63": 2223151104.0, - "64": 2223151104.0, - "65": 2223151104.0, - "66": 2223151104.0, - "67": 2223151104.0, - "68": 2223151104.0, - "69": 2223151104.0, - "70": 2223151104.0, - "71": 2223151104.0, - "72": 2223151104.0, - "73": 2223151104.0, - "74": 2223151104.0, - "75": 2223151104.0, - "76": 2223151104.0, - "77": 2223151104.0, - "78": 2223151104.0, - "79": 2223151104.0, - "80": 2223151104.0, - "81": 2223151104.0, - "82": 2223151104.0, - "83": 2223151104.0, - "84": 2223151104.0, - "85": 2223151104.0, - "86": 2223151104.0, - "87": 2223151104.0, - "88": 2223151104.0, - "89": 2223151104.0, - "90": 2223151104.0, - "91": 2223151104.0, - "92": 2223151104.0, - "93": 2223151104.0, - "94": 2223151104.0, - "95": 2223151104.0, - "96": 2223151104.0, - "97": 2223151104.0, - "98": 2223151104.0, - "99": 2223151104.0, - "100": 2223151104.0 + "1": 1939393536.0, + "2": 2220398592.0, + "3": 2220398592.0, + "4": 2220398592.0, + "5": 2220398592.0, + "6": 2220398592.0, + "7": 2220398592.0, + "8": 2220398592.0, + "9": 2220398592.0, + "10": 2220398592.0, + "11": 2220398592.0, + "12": 2220398592.0, + "13": 2220398592.0, + "14": 2220398592.0, + "15": 2220398592.0, + "16": 2220398592.0, + "17": 2220398592.0, + "18": 2220398592.0, + "19": 2220398592.0, + "20": 2220398592.0, + "21": 2220398592.0, + "22": 2220398592.0, + "23": 2220398592.0, + "24": 2220398592.0, + "25": 2220398592.0, + "26": 2220398592.0, + "27": 2220398592.0, + "28": 2220398592.0, + "29": 2220398592.0, + "30": 2220398592.0, + "31": 2220398592.0, + "32": 2220398592.0, + "33": 2220398592.0, + "34": 2220398592.0, + "35": 2220398592.0, + "36": 2220398592.0, + "37": 2220398592.0, + "38": 2220398592.0, + "39": 2220398592.0, + "40": 2220398592.0, + "41": 2220398592.0, + "42": 2220398592.0, + "43": 2220398592.0, + "44": 2220398592.0, + "45": 2220398592.0, + "46": 2220398592.0, + "47": 2220398592.0, + "48": 2220398592.0, + "49": 2220398592.0, + "50": 2220398592.0, + "51": 2220398592.0, + "52": 2220398592.0, + "53": 2220398592.0, + "54": 2220398592.0, + "55": 2220398592.0, + "56": 2220398592.0, + "57": 2220398592.0, + "58": 2220398592.0, + "59": 2220398592.0, + "60": 2220398592.0, + "61": 2220398592.0, + "62": 2220398592.0, + "63": 2220398592.0, + "64": 2220398592.0, + "65": 2220398592.0, + "66": 2220398592.0, + "67": 2220398592.0, + "68": 2220398592.0, + "69": 2220398592.0, + "70": 2220398592.0, + "71": 2220398592.0, + "72": 2220398592.0, + "73": 2220398592.0, + "74": 2220398592.0, + "75": 2220398592.0, + "76": 2220398592.0, + "77": 2220398592.0, + "78": 2220398592.0, + "79": 2220398592.0, + "80": 2220398592.0, + "81": 2220398592.0, + "82": 2220398592.0, + "83": 2220398592.0, + "84": 2220398592.0, + "85": 2220398592.0, + "86": 2220398592.0, + "87": 2220398592.0, + "88": 2220398592.0, + "89": 2220398592.0, + "90": 2220398592.0, + "91": 2220398592.0, + "92": 2220398592.0, + "93": 2220398592.0, + "94": 2220398592.0, + "95": 2220398592.0, + "96": 2220398592.0, + "97": 2220398592.0, + "98": 2220398592.0, + "99": 2220398592.0, + "100": 2220398592.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.07321, - "3": 0.16613, - "4": 0.14829, - "5": 0.14828, - "6": 0.14699, - "7": 0.14702, - "8": 0.14462, - "9": 0.14411, - "10": 0.14527, - "11": 0.14579, - "12": 0.14583, - "13": 0.14348, - "14": 0.14479, - "15": 0.14514, - "16": 0.14532, - "17": 0.14611, - "18": 0.14647, - "19": 0.14581, - "20": 0.1456, - "21": 0.14543, - "22": 0.14577, - "23": 0.14628, - "24": 0.14522, - "25": 0.1453, - "26": 0.14709, - "27": 0.14625, - "28": 0.14692, - "29": 0.14822, - "30": 0.14481, - "31": 0.1458, - "32": 0.14594, - "33": 0.14696, - "34": 0.14564, - "35": 0.14636, - "36": 0.1473, - "37": 0.14847, - "38": 0.14756, - "39": 0.14685, - "40": 0.14635, - "41": 0.14764, - "42": 0.146, - "43": 0.14662, - "44": 0.14867, - "45": 0.14668, - "46": 0.14504, - "47": 0.14696, - "48": 0.15052, - "49": 0.14764, - "50": 0.14541, - "51": 0.30732, - "52": 0.159, - "53": 0.17959, - "54": 0.14834, - "55": 0.14676, - "56": 0.14782, - "57": 0.1458, - "58": 0.14415, - "59": 0.14544, - "60": 0.14384, - "61": 0.14495, - "62": 0.14607, - "63": 0.14433, - "64": 0.14399, - "65": 0.14521, - "66": 0.14494, - "67": 0.14776, - "68": 0.14658, - "69": 0.14541, - "70": 0.14535, - "71": 0.14351, - "72": 0.14531, - "73": 0.14509, - "74": 0.14566, - "75": 0.14489, - "76": 0.14753, - "77": 0.14428, - "78": 0.1449, - "79": 0.14507, - "80": 0.1452, - "81": 0.14555, - "82": 0.14562, - "83": 0.14649, - "84": 0.14654, - "85": 0.14513, - "86": 0.14655, - "87": 0.14634, - "88": 0.14819, - "89": 0.14809, - "90": 0.14682, - "91": 0.14506, - "92": 0.14539, - "93": 0.14815, - "94": 0.14711, - "95": 0.14727, - "96": 0.14493, - "97": 0.14414, - "98": 0.14933, - "99": 0.14649, - "100": 0.14478 + "2": 8.97862, + "3": 0.16884, + "4": 0.12619, + "5": 0.12723, + "6": 0.12673, + "7": 0.12714, + "8": 0.12536, + "9": 0.12607, + "10": 0.12656, + "11": 0.12642, + "12": 0.1263, + "13": 0.12555, + "14": 0.12971, + "15": 0.1259, + "16": 0.12714, + "17": 0.1256, + "18": 0.12678, + "19": 0.12401, + "20": 0.12894, + "21": 0.1275, + "22": 0.12084, + "23": 0.12458, + "24": 0.12462, + "25": 0.12519, + "26": 0.12824, + "27": 0.12667, + "28": 0.12528, + "29": 0.12522, + "30": 0.12429, + "31": 0.12474, + "32": 0.12623, + "33": 0.1231, + "34": 0.1223, + "35": 0.12293, + "36": 0.12292, + "37": 0.12334, + "38": 0.12614, + "39": 0.12511, + "40": 0.12254, + "41": 0.12361, + "42": 0.1262, + "43": 0.12691, + "44": 0.12331, + "45": 0.12511, + "46": 0.12567, + "47": 0.12375, + "48": 0.1229, + "49": 0.12331, + "50": 0.12537, + "51": 0.32022, + "52": 0.17364, + "53": 0.13187, + "54": 0.12838, + "55": 0.12745, + "56": 0.12305, + "57": 0.12336, + "58": 0.12489, + "59": 0.12486, + "60": 0.12577, + "61": 0.12582, + "62": 0.1238, + "63": 0.12574, + "64": 0.12602, + "65": 0.12514, + "66": 0.12613, + "67": 0.12665, + "68": 0.12591, + "69": 0.2304, + "70": 0.12624, + "71": 0.12509, + "72": 0.12303, + "73": 0.12475, + "74": 0.12483, + "75": 0.12582, + "76": 0.12365, + "77": 0.1253, + "78": 0.12441, + "79": 0.12557, + "80": 0.12522, + "81": 0.12313, + "82": 0.1223, + "83": 0.12472, + "84": 0.12283, + "85": 0.12375, + "86": 0.12309, + "87": 0.11986, + "88": 0.12202, + "89": 0.12407, + "90": 0.12311, + "91": 0.12335, + "92": 0.12578, + "93": 0.12377, + "94": 0.12371, + "95": 0.12221, + "96": 0.12333, + "97": 0.12413, + "98": 0.12292, + "99": 0.1233, + "100": 0.12386 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json index d479cad094f..c9e0bb560d9 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.91715, - "2": 10.91532, - "3": 10.90493, + "2": 10.9154, + "3": 10.90491, "4": 10.90847, - "5": 10.90691, - "6": 10.9055, - "7": 10.92235, + "5": 10.90687, + "6": 10.90548, + "7": 10.92229, "8": 10.90215, - "9": 10.90498, - "10": 10.89605, - "11": 10.90018, - "12": 10.90163, + "9": 10.905, + "10": 10.89603, + "11": 10.90014, + "12": 10.90167, "13": 10.90201, - "14": 10.88651, - "15": 10.89684, - "16": 10.86925, - "17": 10.87976, - "18": 10.84695, - "19": 10.87292, - "20": 10.82075, - "21": 10.82485, - "22": 10.81217, - "23": 10.80616, - "24": 10.76755, - "25": 10.78042, - "26": 10.76321, - "27": 10.7656, - "28": 10.70637, - "29": 10.68618, - "30": 10.66655, - "31": 10.65301, - "32": 10.64628, - "33": 10.61261, - "34": 10.5899, - "35": 10.58928, - "36": 10.56372, - "37": 10.55028, - "38": 10.55748, + "14": 10.88648, + "15": 10.8968, + "16": 10.86928, + "17": 10.87979, + "18": 10.847, + "19": 10.87283, + "20": 10.82063, + "21": 10.82483, + "22": 10.81215, + "23": 10.80624, + "24": 10.76779, + "25": 10.78047, + "26": 10.76316, + "27": 10.76555, + "28": 10.70634, + "29": 10.68625, + "30": 10.66666, + "31": 10.65299, + "32": 10.64627, + "33": 10.61257, + "34": 10.58996, + "35": 10.58931, + "36": 10.5638, + "37": 10.5502, + "38": 10.5576, "39": 10.50946, - "40": 10.49801, - "41": 10.46839, - "42": 10.46037, - "43": 10.42389, - "44": 10.40559, - "45": 10.40957, - "46": 10.36962, - "47": 10.36339, - "48": 10.31729, - "49": 10.31454, - "50": 10.31019, - "51": 10.29764, - "52": 10.25606, - "53": 10.24571, - "54": 10.21517, - "55": 10.18762, + "40": 10.49804, + "41": 10.46843, + "42": 10.46039, + "43": 10.42395, + "44": 10.40554, + "45": 10.40958, + "46": 10.36961, + "47": 10.36341, + "48": 10.31738, + "49": 10.3146, + "50": 10.31027, + "51": 10.29769, + "52": 10.25613, + "53": 10.24574, + "54": 10.21521, + "55": 10.1877, "56": 10.21176, - "57": 10.1873, - "58": 10.20309, - "59": 10.15361, - "60": 10.15374, - "61": 10.12073, - "62": 10.07994, - "63": 10.14838, - "64": 10.09518, - "65": 10.06418, - "66": 10.08944, - "67": 10.06057, - "68": 10.02184, - "69": 10.03115, - "70": 10.02659, - "71": 10.0429, - "72": 10.0198, - "73": 9.99838, - "74": 9.98812, - "75": 9.9503, - "76": 10.00097, - "77": 9.99194, + "57": 10.18736, + "58": 10.20312, + "59": 10.15362, + "60": 10.15372, + "61": 10.12077, + "62": 10.07997, + "63": 10.14849, + "64": 10.09525, + "65": 10.06421, + "66": 10.08945, + "67": 10.06061, + "68": 10.02194, + "69": 10.03116, + "70": 10.02658, + "71": 10.04295, + "72": 10.01976, + "73": 9.99839, + "74": 9.98819, + "75": 9.95031, + "76": 10.00101, + "77": 9.992, "78": 9.93894, - "79": 9.9475, - "80": 9.95199, - "81": 9.97687, - "82": 9.93072, - "83": 9.89056, - "84": 9.82765, - "85": 9.81182, - "86": 9.9222, - "87": 9.94021, - "88": 9.9185, - "89": 9.85561, - "90": 9.85341, - "91": 9.86676, - "92": 9.85465, - "93": 9.79188, - "94": 9.86491, - "95": 9.86283, - "96": 9.84804, - "97": 9.78916, - "98": 9.82351, - "99": 9.86502, - "100": 9.76301 + "79": 9.94753, + "80": 9.95206, + "81": 9.97693, + "82": 9.93076, + "83": 9.89058, + "84": 9.82768, + "85": 9.81185, + "86": 9.92221, + "87": 9.94022, + "88": 9.91853, + "89": 9.8556, + "90": 9.85347, + "91": 9.8667, + "92": 9.85471, + "93": 9.79187, + "94": 9.8649, + "95": 9.86278, + "96": 9.8481, + "97": 9.78919, + "98": 9.82356, + "99": 9.86505, + "100": 9.76305 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2974.0, - "2": 3007.0, - "3": 2774.0, - "4": 2880.0, - "5": 2917.0, - "6": 2797.0, - "7": 2954.0, - "8": 2881.0, - "9": 3016.0, - "10": 3002.0, - "11": 3025.0, - "12": 2876.0, - "13": 3098.0, - "14": 3056.0, - "15": 2793.0, - "16": 2980.0, - "17": 2895.0, - "18": 2749.0, - "19": 3152.0, - "20": 2925.0, - "21": 2901.0, - "22": 2845.0, - "23": 2802.0, - "24": 2832.0, - "25": 3001.0, - "26": 2853.0, - "27": 2791.0, - "28": 2843.0, - "29": 2785.0, - "30": 2711.0, - "31": 2699.0, - "32": 2647.0, - "33": 2622.0, - "34": 2823.0, - "35": 2677.0, - "36": 2661.0, - "37": 2791.0, - "38": 2749.0, - "39": 2543.0, - "40": 2735.0, - "41": 2547.0, - "42": 2688.0, - "43": 2594.0, - "44": 2635.0, - "45": 2613.0, - "46": 2635.0, - "47": 2571.0, - "48": 2605.0, - "49": 2710.0, - "50": 2619.0, - "51": 2728.0, - "52": 2683.0, - "53": 2654.0, - "54": 2822.0, - "55": 2512.0, - "56": 2942.0, - "57": 2461.0, - "58": 3258.0, - "59": 2858.0, - "60": 2770.0, - "61": 2863.0, - "62": 2811.0, - "63": 3004.0, - "64": 3008.0, - "65": 2584.0, - "66": 2961.0, - "67": 3223.0, - "68": 2963.0, - "69": 3022.0, - "70": 2992.0, - "71": 2761.0, - "72": 3101.0, - "73": 3109.0, - "74": 2917.0, - "75": 2949.0, - "76": 3185.0, - "77": 3265.0, - "78": 3091.0, - "79": 3067.0, - "80": 3041.0, - "81": 3235.0, - "82": 3487.0, - "83": 3187.0, - "84": 3089.0, - "85": 2761.0, - "86": 3019.0, - "87": 3151.0, - "88": 3283.0, - "89": 3203.0, - "90": 3469.0, - "91": 2839.0, - "92": 3056.0, - "93": 3055.0, - "94": 3245.0, - "95": 3213.0, - "96": 3280.0, - "97": 3508.0, - "98": 3586.0, - "99": 3138.0, - "100": 3337.0 + "1": 2918.0, + "2": 3102.0, + "3": 2838.0, + "4": 3016.0, + "5": 2866.0, + "6": 2736.0, + "7": 2941.0, + "8": 2853.0, + "9": 3050.0, + "10": 3021.0, + "11": 3044.0, + "12": 2869.0, + "13": 3184.0, + "14": 3101.0, + "15": 2827.0, + "16": 3146.0, + "17": 2960.0, + "18": 2676.0, + "19": 3031.0, + "20": 2898.0, + "21": 2961.0, + "22": 2898.0, + "23": 2698.0, + "24": 2874.0, + "25": 3055.0, + "26": 2787.0, + "27": 2867.0, + "28": 2832.0, + "29": 2835.0, + "30": 2845.0, + "31": 2598.0, + "32": 2709.0, + "33": 2551.0, + "34": 2674.0, + "35": 2666.0, + "36": 2641.0, + "37": 2633.0, + "38": 2709.0, + "39": 2502.0, + "40": 2702.0, + "41": 2519.0, + "42": 2662.0, + "43": 2706.0, + "44": 2529.0, + "45": 2729.0, + "46": 2625.0, + "47": 2637.0, + "48": 2644.0, + "49": 2640.0, + "50": 2512.0, + "51": 2810.0, + "52": 2735.0, + "53": 2596.0, + "54": 2751.0, + "55": 2550.0, + "56": 2895.0, + "57": 2555.0, + "58": 3169.0, + "59": 2771.0, + "60": 2744.0, + "61": 2896.0, + "62": 2917.0, + "63": 3047.0, + "64": 3092.0, + "65": 2659.0, + "66": 2953.0, + "67": 3177.0, + "68": 3018.0, + "69": 3070.0, + "70": 3010.0, + "71": 2935.0, + "72": 3099.0, + "73": 3172.0, + "74": 3012.0, + "75": 2879.0, + "76": 3272.0, + "77": 3240.0, + "78": 3120.0, + "79": 3224.0, + "80": 2985.0, + "81": 3310.0, + "82": 3405.0, + "83": 3033.0, + "84": 3023.0, + "85": 2821.0, + "86": 2995.0, + "87": 3146.0, + "88": 3236.0, + "89": 3216.0, + "90": 3396.0, + "91": 2877.0, + "92": 3067.0, + "93": 3052.0, + "94": 3194.0, + "95": 3059.0, + "96": 3288.0, + "97": 3526.0, + "98": 3653.0, + "99": 3087.0, + "100": 3208.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.68703, - "3": 2.27607, - "4": 1.95356, - "5": 1.43808, - "6": 1.66974, - "7": 1.82073, - "8": 2.45573, - "9": 2.45608, - "10": 1.88881, - "11": 2.2115, - "12": 1.76382, - "13": 1.7887, - "14": 1.94066, - "15": 1.87084, - "16": 1.8839, - "17": 2.15134, - "18": 1.87321, - "19": 2.00269, - "20": 1.8935, - "21": 2.23123, - "22": 1.68075, - "23": 2.2295, - "24": 1.89249, - "25": 2.04549, - "26": 2.23751, - "27": 1.97092, - "28": 1.87329, - "29": 1.65388, - "30": 1.64312, - "31": 1.58189, - "32": 2.71233, - "33": 1.54317, - "34": 1.52133, - "35": 2.36052, - "36": 2.12589, - "37": 1.87697, - "38": 1.68171, - "39": 2.02993, - "40": 1.85575, - "41": 1.76289, - "42": 2.56863, - "43": 1.98611, - "44": 1.60222, - "45": 2.34358, - "46": 1.89547, - "47": 1.77595, - "48": 2.16868, - "49": 1.64316, - "50": 2.07621, - "51": 0.91935, - "52": 2.4302, - "53": 1.77192, - "54": 1.93281, - "55": 1.75761, - "56": 2.85699, - "57": 1.44381, - "58": 2.09848, - "59": 1.81572, - "60": 1.71216, - "61": 2.07107, - "62": 2.19075, - "63": 1.54832, - "64": 1.95479, - "65": 1.68437, - "66": 2.35293, - "67": 2.04743, - "68": 1.54269, - "69": 1.95756, - "70": 2.02158, - "71": 2.22677, - "72": 2.15266, - "73": 1.74043, - "74": 1.65764, - "75": 1.73174, - "76": 2.24875, - "77": 1.71524, - "78": 1.81966, - "79": 2.05485, - "80": 1.99433, - "81": 2.21734, - "82": 2.06341, - "83": 2.00711, - "84": 1.65729, - "85": 1.88703, - "86": 1.77847, - "87": 2.37983, - "88": 1.89045, - "89": 2.40014, - "90": 2.14625, - "91": 2.02425, - "92": 2.18983, - "93": 1.43472, - "94": 1.92868, - "95": 1.85937, - "96": 2.18236, - "97": 1.76772, - "98": 1.93071, - "99": 2.37598, - "100": 1.72405 + "2": 7.01992, + "3": 2.10309, + "4": 1.92125, + "5": 1.71359, + "6": 1.85526, + "7": 1.64311, + "8": 2.00641, + "9": 2.29604, + "10": 1.96922, + "11": 1.86114, + "12": 1.34002, + "13": 1.91381, + "14": 1.85994, + "15": 1.57169, + "16": 1.86321, + "17": 2.14456, + "18": 1.41602, + "19": 1.57945, + "20": 1.77837, + "21": 2.07281, + "22": 1.47715, + "23": 1.40181, + "24": 1.883, + "25": 1.93325, + "26": 2.42456, + "27": 1.52734, + "28": 1.7609, + "29": 1.2204, + "30": 1.65488, + "31": 1.40104, + "32": 2.50875, + "33": 1.25261, + "34": 1.26214, + "35": 1.75784, + "36": 1.63659, + "37": 1.84814, + "38": 1.17309, + "39": 2.11569, + "40": 1.76873, + "41": 1.93865, + "42": 2.10627, + "43": 1.71518, + "44": 1.49533, + "45": 1.7238, + "46": 1.83904, + "47": 1.69925, + "48": 2.30835, + "49": 1.71346, + "50": 2.27987, + "51": 0.97177, + "52": 2.06976, + "53": 1.94804, + "54": 1.99391, + "55": 1.76935, + "56": 2.34679, + "57": 1.04144, + "58": 1.93804, + "59": 1.38455, + "60": 1.43609, + "61": 1.9623, + "62": 1.95464, + "63": 1.40352, + "64": 1.70073, + "65": 1.37597, + "66": 1.85585, + "67": 1.86451, + "68": 1.35819, + "69": 1.91056, + "70": 1.74015, + "71": 2.24564, + "72": 1.75621, + "73": 1.59018, + "74": 1.50085, + "75": 1.77801, + "76": 1.72109, + "77": 1.68959, + "78": 1.73278, + "79": 1.98931, + "80": 1.92583, + "81": 1.62238, + "82": 2.31819, + "83": 1.8841, + "84": 1.70588, + "85": 1.58672, + "86": 1.75002, + "87": 1.94629, + "88": 1.76342, + "89": 1.50976, + "90": 1.72391, + "91": 1.55028, + "92": 1.87252, + "93": 1.53436, + "94": 1.77771, + "95": 2.01535, + "96": 1.73826, + "97": 1.81918, + "98": 1.69215, + "99": 2.21317, + "100": 1.457 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json index 3617fe6ec66..0c937c58793 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json @@ -4,105 +4,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94936, - "2": 10.94427, - "3": 10.94489, - "4": 10.94497, - "5": 10.94952, - "6": 10.95081, - "7": 10.93655, - "8": 10.94777, - "9": 10.93487, - "10": 10.94147, - "11": 10.93179, - "12": 10.94114, - "13": 10.90624, - "14": 10.92044, - "15": 10.89675, - "16": 10.89602, - "17": 10.88821, + "1": 10.94938, + "2": 10.94425, + "3": 10.94496, + "4": 10.94498, + "5": 10.9495, + "6": 10.95073, + "7": 10.93652, + "8": 10.94776, + "9": 10.93488, + "10": 10.94139, + "11": 10.93178, + "12": 10.9411, + "13": 10.90625, + "14": 10.9205, + "15": 10.89678, + "16": 10.89609, + "17": 10.88817, "18": 10.86718, - "19": 10.87257, - "20": 10.81918, - "21": 10.78118, + "19": 10.87253, + "20": 10.81921, + "21": 10.78124, "22": 10.76922, - "23": 10.77114, - "24": 10.74817, - "25": 10.74711, - "26": 10.71935, - "27": 10.67537, - "28": 10.60363, - "29": 10.57162, - "30": 10.55937, - "31": 10.54344, - "32": 10.52916, - "33": 10.48537, - "34": 10.48002, - "35": 10.45793, - "36": 10.44314, - "37": 10.407, - "38": 10.40601, - "39": 10.36639, - "40": 10.36706, - "41": 10.33575, - "42": 10.30508, - "43": 10.28584, - "44": 10.25576, - "45": 10.26901, - "46": 10.22099, - "47": 10.20947, - "48": 10.19007, - "49": 10.17535, - "50": 10.16762, - "51": 10.16416, - "52": 10.12594, - "53": 10.12768, - "54": 10.09552, - "55": 10.0646, - "56": 10.07532, - "57": 10.07111, - "58": 10.08237, - "59": 10.02072, - "60": 10.04257, - "61": 10.00327, - "62": 9.97346, - "63": 10.0526, - "64": 10.00611, - "65": 9.97942, - "66": 9.99817, - "67": 9.96025, - "68": 9.93422, - "69": 9.97282, - "70": 9.94456, - "71": 9.96473, - "72": 9.94635, - "73": 9.92086, - "74": 9.91484, - "75": 9.88376, - "76": 9.9241, - "77": 9.90406, - "78": 9.86835, - "79": 9.86617, - "80": 9.87858, - "81": 9.89301, - "82": 9.82637, - "83": 9.80163, - "84": 9.74865, - "85": 9.72929, - "86": 9.84445, - "87": 9.85641, - "88": 9.83837, - "89": 9.77534, - "90": 9.76247, - "91": 9.78205, - "92": 9.7546, - "93": 9.70175, - "94": 9.77726, - "95": 9.76596, - "96": 9.76819, - "97": 9.70107, - "98": 9.72321, - "99": 9.75933, + "23": 10.77115, + "24": 10.74809, + "25": 10.74714, + "26": 10.71941, + "27": 10.67539, + "28": 10.60365, + "29": 10.57167, + "30": 10.55935, + "31": 10.54345, + "32": 10.52918, + "33": 10.48539, + "34": 10.48006, + "35": 10.45794, + "36": 10.44315, + "37": 10.40701, + "38": 10.40609, + "39": 10.36645, + "40": 10.36709, + "41": 10.33576, + "42": 10.30509, + "43": 10.28581, + "44": 10.25578, + "45": 10.26907, + "46": 10.22104, + "47": 10.20952, + "48": 10.19017, + "49": 10.17541, + "50": 10.16767, + "51": 10.16425, + "52": 10.12595, + "53": 10.1277, + "54": 10.09557, + "55": 10.06458, + "56": 10.0754, + "57": 10.07117, + "58": 10.0824, + "59": 10.02077, + "60": 10.04262, + "61": 10.0033, + "62": 9.97352, + "63": 10.05265, + "64": 10.00618, + "65": 9.97947, + "66": 9.99818, + "67": 9.96027, + "68": 9.93423, + "69": 9.97285, + "70": 9.94462, + "71": 9.96475, + "72": 9.94639, + "73": 9.9209, + "74": 9.91486, + "75": 9.88383, + "76": 9.92417, + "77": 9.90414, + "78": 9.86839, + "79": 9.86625, + "80": 9.8786, + "81": 9.89309, + "82": 9.82639, + "83": 9.80165, + "84": 9.74873, + "85": 9.72934, + "86": 9.84449, + "87": 9.85644, + "88": 9.83839, + "89": 9.77541, + "90": 9.76252, + "91": 9.78211, + "92": 9.75465, + "93": 9.70179, + "94": 9.77729, + "95": 9.766, + "96": 9.76825, + "97": 9.70112, + "98": 9.72329, + "99": 9.75935, "100": 9.67061 } }, @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22962096.0, - "2": 22850140.0, - "3": 22708724.0, - "4": 22777762.0, - "5": 22783424.0, - "6": 22746224.0, - "7": 22871734.0, - "8": 22611322.0, - "9": 22758224.0, - "10": 22489136.0, - "11": 22753716.0, - "12": 22640368.0, - "13": 23322530.0, - "14": 22992254.0, - "15": 22718080.0, - "16": 22822500.0, - "17": 22929124.0, - "18": 22994868.0, - "19": 23084620.0, - "20": 22722880.0, - "21": 22918336.0, - "22": 22944758.0, - "23": 22629448.0, - "24": 22861824.0, - "25": 22635548.0, - "26": 23004368.0, - "27": 22802780.0, - "28": 22998908.0, - "29": 22979892.0, - "30": 22943770.0, - "31": 22907970.0, - "32": 22666452.0, - "33": 22740660.0, - "34": 23075832.0, - "35": 22750476.0, - "36": 22695416.0, - "37": 23103956.0, - "38": 22964648.0, - "39": 22985528.0, - "40": 22750002.0, - "41": 23063976.0, - "42": 22688524.0, - "43": 22987628.0, - "44": 22704872.0, - "45": 22846896.0, - "46": 22734028.0, - "47": 22849208.0, - "48": 22833136.0, - "49": 22888792.0, - "50": 22644196.0, - "51": 22698844.0, - "52": 22812860.0, - "53": 22962368.0, - "54": 22786320.0, - "55": 22922736.0, - "56": 22659552.0, - "57": 23193148.0, - "58": 22688238.0, - "59": 22841772.0, - "60": 23014078.0, - "61": 22673244.0, - "62": 22732404.0, - "63": 22632150.0, - "64": 22999482.0, - "65": 23198800.0, - "66": 22694190.0, - "67": 22956416.0, - "68": 22924652.0, - "69": 23150510.0, - "70": 22822026.0, - "71": 22739532.0, - "72": 23116668.0, - "73": 23133724.0, - "74": 22941132.0, - "75": 22880338.0, - "76": 22702428.0, - "77": 22980378.0, - "78": 22978056.0, - "79": 22823958.0, - "80": 22930598.0, - "81": 22834428.0, - "82": 22730884.0, - "83": 22734844.0, - "84": 23099080.0, - "85": 22922516.0, - "86": 23069004.0, - "87": 22374000.0, - "88": 22555776.0, - "89": 22727918.0, - "90": 22765904.0, - "91": 22919272.0, - "92": 22672496.0, - "93": 22645720.0, - "94": 23128452.0, - "95": 22694664.0, - "96": 22845610.0, - "97": 22831708.0, - "98": 22875336.0, - "99": 22637086.0, - "100": 23000228.0 + "1": 22962064.0, + "2": 22850056.0, + "3": 22708664.0, + "4": 22777884.0, + "5": 22783416.0, + "6": 22746210.0, + "7": 22871732.0, + "8": 22611404.0, + "9": 22758118.0, + "10": 22489220.0, + "11": 22753708.0, + "12": 22640420.0, + "13": 23322488.0, + "14": 22992200.0, + "15": 22718100.0, + "16": 22822436.0, + "17": 22929100.0, + "18": 22994840.0, + "19": 23084644.0, + "20": 22722896.0, + "21": 22918344.0, + "22": 22944722.0, + "23": 22629320.0, + "24": 22861788.0, + "25": 22635592.0, + "26": 23004344.0, + "27": 22802814.0, + "28": 22998928.0, + "29": 22979896.0, + "30": 22943594.0, + "31": 22907932.0, + "32": 22666402.0, + "33": 22740754.0, + "34": 23075746.0, + "35": 22750456.0, + "36": 22695508.0, + "37": 23103922.0, + "38": 22964604.0, + "39": 22985592.0, + "40": 22749904.0, + "41": 23063922.0, + "42": 22688558.0, + "43": 22987592.0, + "44": 22704836.0, + "45": 22846894.0, + "46": 22733946.0, + "47": 22849304.0, + "48": 22833182.0, + "49": 22888720.0, + "50": 22644266.0, + "51": 22698956.0, + "52": 22812846.0, + "53": 22962328.0, + "54": 22786188.0, + "55": 22922802.0, + "56": 22659634.0, + "57": 23193328.0, + "58": 22688208.0, + "59": 22841704.0, + "60": 23014132.0, + "61": 22673272.0, + "62": 22732548.0, + "63": 22632228.0, + "64": 22999512.0, + "65": 23198706.0, + "66": 22694180.0, + "67": 22956284.0, + "68": 22924760.0, + "69": 23150400.0, + "70": 22822050.0, + "71": 22739580.0, + "72": 23116486.0, + "73": 23133682.0, + "74": 22941040.0, + "75": 22880268.0, + "76": 22702468.0, + "77": 22980482.0, + "78": 22978144.0, + "79": 22823910.0, + "80": 22930772.0, + "81": 22834530.0, + "82": 22730778.0, + "83": 22734652.0, + "84": 23098968.0, + "85": 22922592.0, + "86": 23069212.0, + "87": 22374152.0, + "88": 22555580.0, + "89": 22727970.0, + "90": 22765920.0, + "91": 22919114.0, + "92": 22672400.0, + "93": 22645840.0, + "94": 23128392.0, + "95": 22694746.0, + "96": 22845536.0, + "97": 22831780.0, + "98": 22875492.0, + "99": 22636892.0, + "100": 23000280.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 748541440.0, - "2": 748541440.0, - "3": 748541440.0, - "4": 748541440.0, - "5": 748541440.0, - "6": 748541440.0, - "7": 748541440.0, - "8": 748541440.0, - "9": 748541440.0, - "10": 748541440.0, - "11": 748541440.0, - "12": 748541440.0, - "13": 748541440.0, - "14": 748541440.0, - "15": 748541440.0, - "16": 748541440.0, - "17": 748541440.0, - "18": 748541440.0, - "19": 748541440.0, - "20": 748541440.0, - "21": 748541440.0, - "22": 748541440.0, - "23": 748541440.0, - "24": 748541440.0, - "25": 748541440.0, - "26": 748541440.0, - "27": 748541440.0, - "28": 748541440.0, - "29": 748541440.0, - "30": 748541440.0, - "31": 748541440.0, - "32": 748541440.0, - "33": 748541440.0, - "34": 748541440.0, - "35": 748541440.0, - "36": 748541440.0, - "37": 748541440.0, - "38": 748541440.0, - "39": 748541440.0, - "40": 748541440.0, - "41": 748541440.0, - "42": 748541440.0, - "43": 748541440.0, - "44": 748541440.0, - "45": 748541440.0, - "46": 748541440.0, - "47": 748541440.0, - "48": 748541440.0, - "49": 748541440.0, - "50": 748541440.0, - "51": 748541440.0, - "52": 748541440.0, - "53": 748541440.0, - "54": 748541440.0, - "55": 748541440.0, - "56": 748541440.0, - "57": 748541440.0, - "58": 748541440.0, - "59": 748541440.0, - "60": 748541440.0, - "61": 748541440.0, - "62": 748541440.0, - "63": 748541440.0, - "64": 748541440.0, - "65": 748541440.0, - "66": 748541440.0, - "67": 748541440.0, - "68": 748541440.0, - "69": 748541440.0, - "70": 748541440.0, - "71": 748541440.0, - "72": 748541440.0, - "73": 748541440.0, - "74": 748541440.0, - "75": 748541440.0, - "76": 748541440.0, - "77": 748541440.0, - "78": 748541440.0, - "79": 748541440.0, - "80": 748541440.0, - "81": 748541440.0, - "82": 748541440.0, - "83": 748541440.0, - "84": 748541440.0, - "85": 748541440.0, - "86": 748541440.0, - "87": 748541440.0, - "88": 748541440.0, - "89": 748541440.0, - "90": 748541440.0, - "91": 748541440.0, - "92": 748541440.0, - "93": 748541440.0, - "94": 748541440.0, - "95": 748541440.0, - "96": 748541440.0, - "97": 748541440.0, - "98": 748541440.0, - "99": 748541440.0, - "100": 748541440.0 + "1": 746444288.0, + "2": 746444288.0, + "3": 746444288.0, + "4": 746444288.0, + "5": 746444288.0, + "6": 746444288.0, + "7": 746444288.0, + "8": 746444288.0, + "9": 746444288.0, + "10": 746444288.0, + "11": 746444288.0, + "12": 746444288.0, + "13": 746444288.0, + "14": 746444288.0, + "15": 746444288.0, + "16": 746444288.0, + "17": 746444288.0, + "18": 746444288.0, + "19": 746444288.0, + "20": 746444288.0, + "21": 746444288.0, + "22": 746444288.0, + "23": 746444288.0, + "24": 746444288.0, + "25": 746444288.0, + "26": 746444288.0, + "27": 746444288.0, + "28": 746444288.0, + "29": 746444288.0, + "30": 746444288.0, + "31": 746444288.0, + "32": 746444288.0, + "33": 746444288.0, + "34": 746444288.0, + "35": 746444288.0, + "36": 746444288.0, + "37": 746444288.0, + "38": 746444288.0, + "39": 746444288.0, + "40": 746444288.0, + "41": 746444288.0, + "42": 746444288.0, + "43": 746444288.0, + "44": 746444288.0, + "45": 746444288.0, + "46": 746444288.0, + "47": 746444288.0, + "48": 746444288.0, + "49": 746444288.0, + "50": 746444288.0, + "51": 746444288.0, + "52": 746444288.0, + "53": 746444288.0, + "54": 746444288.0, + "55": 746444288.0, + "56": 746444288.0, + "57": 746444288.0, + "58": 746444288.0, + "59": 746444288.0, + "60": 746444288.0, + "61": 746444288.0, + "62": 746444288.0, + "63": 746444288.0, + "64": 746444288.0, + "65": 746444288.0, + "66": 746444288.0, + "67": 746444288.0, + "68": 746444288.0, + "69": 746444288.0, + "70": 746444288.0, + "71": 746444288.0, + "72": 746444288.0, + "73": 746444288.0, + "74": 746444288.0, + "75": 746444288.0, + "76": 746444288.0, + "77": 746444288.0, + "78": 746444288.0, + "79": 746444288.0, + "80": 746444288.0, + "81": 746444288.0, + "82": 746444288.0, + "83": 746444288.0, + "84": 746444288.0, + "85": 746444288.0, + "86": 746444288.0, + "87": 746444288.0, + "88": 746444288.0, + "89": 746444288.0, + "90": 746444288.0, + "91": 746444288.0, + "92": 746444288.0, + "93": 746444288.0, + "94": 746444288.0, + "95": 746444288.0, + "96": 746444288.0, + "97": 746444288.0, + "98": 746444288.0, + "99": 746444288.0, + "100": 746444288.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1939923968.0, - "2": 2224781312.0, - "3": 2224781312.0, - "4": 2224781312.0, - "5": 2224781312.0, - "6": 2224781312.0, - "7": 2224781312.0, - "8": 2224781312.0, - "9": 2224781312.0, - "10": 2224781312.0, - "11": 2224781312.0, - "12": 2224781312.0, - "13": 2224781312.0, - "14": 2224781312.0, - "15": 2224781312.0, - "16": 2224781312.0, - "17": 2224781312.0, - "18": 2224781312.0, - "19": 2224781312.0, - "20": 2224781312.0, - "21": 2224781312.0, - "22": 2224781312.0, - "23": 2224781312.0, - "24": 2224781312.0, - "25": 2224781312.0, - "26": 2224781312.0, - "27": 2224781312.0, - "28": 2224781312.0, - "29": 2224781312.0, - "30": 2224781312.0, - "31": 2224781312.0, - "32": 2224781312.0, - "33": 2224781312.0, - "34": 2224781312.0, - "35": 2224781312.0, - "36": 2224781312.0, - "37": 2224781312.0, - "38": 2224781312.0, - "39": 2224781312.0, - "40": 2224781312.0, - "41": 2224781312.0, - "42": 2224781312.0, - "43": 2224781312.0, - "44": 2224781312.0, - "45": 2224781312.0, - "46": 2224781312.0, - "47": 2224781312.0, - "48": 2224781312.0, - "49": 2224781312.0, - "50": 2224781312.0, - "51": 2224781312.0, - "52": 2224781312.0, - "53": 2224781312.0, - "54": 2224781312.0, - "55": 2224781312.0, - "56": 2224781312.0, - "57": 2224781312.0, - "58": 2224781312.0, - "59": 2224781312.0, - "60": 2224781312.0, - "61": 2224781312.0, - "62": 2224781312.0, - "63": 2224781312.0, - "64": 2224781312.0, - "65": 2224781312.0, - "66": 2224781312.0, - "67": 2224781312.0, - "68": 2224781312.0, - "69": 2224781312.0, - "70": 2224781312.0, - "71": 2224781312.0, - "72": 2224781312.0, - "73": 2224781312.0, - "74": 2224781312.0, - "75": 2224781312.0, - "76": 2224781312.0, - "77": 2224781312.0, - "78": 2224781312.0, - "79": 2224781312.0, - "80": 2224781312.0, - "81": 2224781312.0, - "82": 2224781312.0, - "83": 2224781312.0, - "84": 2224781312.0, - "85": 2224781312.0, - "86": 2224781312.0, - "87": 2224781312.0, - "88": 2224781312.0, - "89": 2224781312.0, - "90": 2224781312.0, - "91": 2224781312.0, - "92": 2224781312.0, - "93": 2224781312.0, - "94": 2224781312.0, - "95": 2224781312.0, - "96": 2224781312.0, - "97": 2224781312.0, - "98": 2224781312.0, - "99": 2224781312.0, - "100": 2224781312.0 + "1": 1938875392.0, + "2": 2222684160.0, + "3": 2222684160.0, + "4": 2222684160.0, + "5": 2222684160.0, + "6": 2222684160.0, + "7": 2222684160.0, + "8": 2222684160.0, + "9": 2222684160.0, + "10": 2222684160.0, + "11": 2222684160.0, + "12": 2222684160.0, + "13": 2222684160.0, + "14": 2222684160.0, + "15": 2222684160.0, + "16": 2222684160.0, + "17": 2222684160.0, + "18": 2222684160.0, + "19": 2222684160.0, + "20": 2222684160.0, + "21": 2222684160.0, + "22": 2222684160.0, + "23": 2222684160.0, + "24": 2222684160.0, + "25": 2222684160.0, + "26": 2222684160.0, + "27": 2222684160.0, + "28": 2222684160.0, + "29": 2222684160.0, + "30": 2222684160.0, + "31": 2222684160.0, + "32": 2222684160.0, + "33": 2222684160.0, + "34": 2222684160.0, + "35": 2222684160.0, + "36": 2222684160.0, + "37": 2222684160.0, + "38": 2222684160.0, + "39": 2222684160.0, + "40": 2222684160.0, + "41": 2222684160.0, + "42": 2222684160.0, + "43": 2222684160.0, + "44": 2222684160.0, + "45": 2222684160.0, + "46": 2222684160.0, + "47": 2222684160.0, + "48": 2222684160.0, + "49": 2222684160.0, + "50": 2222684160.0, + "51": 2222684160.0, + "52": 2222684160.0, + "53": 2222684160.0, + "54": 2222684160.0, + "55": 2222684160.0, + "56": 2222684160.0, + "57": 2222684160.0, + "58": 2222684160.0, + "59": 2222684160.0, + "60": 2222684160.0, + "61": 2222684160.0, + "62": 2222684160.0, + "63": 2222684160.0, + "64": 2222684160.0, + "65": 2222684160.0, + "66": 2222684160.0, + "67": 2222684160.0, + "68": 2222684160.0, + "69": 2222684160.0, + "70": 2222684160.0, + "71": 2222684160.0, + "72": 2222684160.0, + "73": 2222684160.0, + "74": 2222684160.0, + "75": 2222684160.0, + "76": 2222684160.0, + "77": 2222684160.0, + "78": 2222684160.0, + "79": 2222684160.0, + "80": 2222684160.0, + "81": 2222684160.0, + "82": 2222684160.0, + "83": 2222684160.0, + "84": 2222684160.0, + "85": 2222684160.0, + "86": 2222684160.0, + "87": 2222684160.0, + "88": 2222684160.0, + "89": 2222684160.0, + "90": 2222684160.0, + "91": 2222684160.0, + "92": 2222684160.0, + "93": 2222684160.0, + "94": 2222684160.0, + "95": 2222684160.0, + "96": 2222684160.0, + "97": 2222684160.0, + "98": 2222684160.0, + "99": 2222684160.0, + "100": 2222684160.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.96547, - "3": 0.15428, - "4": 0.14141, - "5": 0.13859, - "6": 0.13778, - "7": 0.14189, - "8": 0.14193, - "9": 0.1379, - "10": 0.14024, - "11": 0.14143, - "12": 0.13965, - "13": 0.14049, - "14": 0.14043, - "15": 0.14104, - "16": 0.14169, - "17": 0.14303, - "18": 0.14234, - "19": 0.13987, - "20": 0.14008, - "21": 0.14103, - "22": 0.14126, - "23": 0.14104, - "24": 0.14107, - "25": 0.14185, - "26": 0.14173, - "27": 0.14226, - "28": 0.14143, - "29": 0.14098, - "30": 0.13918, - "31": 0.13937, - "32": 0.13852, - "33": 0.13902, - "34": 0.1405, - "35": 0.14083, - "36": 0.13965, - "37": 0.14037, - "38": 0.13964, - "39": 0.14153, - "40": 0.13994, - "41": 0.13914, - "42": 0.13746, - "43": 0.13905, - "44": 0.14032, - "45": 0.13987, - "46": 0.14039, - "47": 0.14054, - "48": 0.14101, - "49": 0.14064, - "50": 0.13954, - "51": 0.30251, - "52": 0.16277, - "53": 0.18057, - "54": 0.14302, - "55": 0.13853, - "56": 0.13885, - "57": 0.13936, - "58": 0.14062, - "59": 0.1414, - "60": 0.13999, - "61": 0.13971, - "62": 0.14116, - "63": 0.14046, - "64": 0.14103, - "65": 0.14024, - "66": 0.13855, - "67": 0.14041, - "68": 0.14073, - "69": 0.13868, - "70": 0.14313, - "71": 0.13915, - "72": 0.13945, - "73": 0.14002, - "74": 0.14112, - "75": 0.14057, - "76": 0.13826, - "77": 0.14021, - "78": 0.14067, - "79": 0.14008, - "80": 0.1384, - "81": 0.14086, - "82": 0.13991, - "83": 0.1404, - "84": 0.14011, - "85": 0.1407, - "86": 0.14136, - "87": 0.13842, - "88": 0.13874, - "89": 0.1388, - "90": 0.13857, - "91": 0.13895, - "92": 0.14083, - "93": 0.13969, - "94": 0.13902, - "95": 0.14114, - "96": 0.14059, - "97": 0.13916, - "98": 0.13904, - "99": 0.13854, - "100": 0.13963 + "2": 7.76332, + "3": 0.16944, + "4": 0.12764, + "5": 0.1272, + "6": 0.12429, + "7": 0.12385, + "8": 0.12161, + "9": 0.1219, + "10": 0.12537, + "11": 0.12518, + "12": 0.12527, + "13": 0.12477, + "14": 0.12697, + "15": 0.12571, + "16": 0.12765, + "17": 0.1266, + "18": 0.12538, + "19": 0.2352, + "20": 0.12303, + "21": 0.12307, + "22": 0.21902, + "23": 0.12631, + "24": 0.12422, + "25": 0.12672, + "26": 0.12492, + "27": 0.12553, + "28": 0.12738, + "29": 0.1269, + "30": 0.12569, + "31": 0.12644, + "32": 0.15796, + "33": 0.12806, + "34": 0.12624, + "35": 0.12566, + "36": 0.12931, + "37": 0.13374, + "38": 0.13148, + "39": 0.12907, + "40": 0.13096, + "41": 0.1286, + "42": 0.12898, + "43": 0.12887, + "44": 0.12837, + "45": 0.13377, + "46": 0.12801, + "47": 0.12544, + "48": 0.12694, + "49": 0.12786, + "50": 0.12699, + "51": 0.29817, + "52": 0.17239, + "53": 0.23221, + "54": 0.12793, + "55": 0.12873, + "56": 0.12708, + "57": 0.12801, + "58": 0.13016, + "59": 0.12703, + "60": 0.12824, + "61": 0.12604, + "62": 0.12987, + "63": 0.12925, + "64": 0.13204, + "65": 0.12884, + "66": 0.12771, + "67": 0.12882, + "68": 0.1282, + "69": 0.13135, + "70": 0.13026, + "71": 0.12967, + "72": 0.12939, + "73": 0.1277, + "74": 0.12861, + "75": 0.1283, + "76": 0.12807, + "77": 0.13116, + "78": 0.12652, + "79": 0.12691, + "80": 0.12901, + "81": 0.12903, + "82": 0.18548, + "83": 0.12931, + "84": 0.12982, + "85": 0.13007, + "86": 0.12954, + "87": 0.12793, + "88": 0.13123, + "89": 0.12977, + "90": 0.1301, + "91": 0.13026, + "92": 0.13107, + "93": 0.13059, + "94": 0.13026, + "95": 0.13227, + "96": 0.13068, + "97": 0.13032, + "98": 0.13079, + "99": 0.13001, + "100": 0.13057 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json index 68f212b9a01..f98e2a45032 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9543, - "2": 10.97008, - "3": 10.96655, - "4": 10.96406, - "5": 10.96112, - "6": 10.95417, - "7": 10.9563, - "8": 10.95685, + "1": 10.95431, + "2": 10.97013, + "3": 10.96654, + "4": 10.96407, + "5": 10.96107, + "6": 10.9542, + "7": 10.95647, + "8": 10.95687, "9": 10.96029, - "10": 10.95842, - "11": 10.95613, - "12": 10.93265, - "13": 10.95939, - "14": 10.94942, - "15": 10.92043, - "16": 10.9, - "17": 10.9186, - "18": 10.90008, - "19": 10.91326, - "20": 10.8208, - "21": 10.81358, - "22": 10.80493, - "23": 10.78963, - "24": 10.75475, - "25": 10.75922, - "26": 10.7484, - "27": 10.69712, - "28": 10.6374, - "29": 10.61525, - "30": 10.5879, - "31": 10.5788, - "32": 10.55825, - "33": 10.52366, - "34": 10.50297, + "10": 10.95843, + "11": 10.95616, + "12": 10.93269, + "13": 10.95941, + "14": 10.94943, + "15": 10.92042, + "16": 10.89999, + "17": 10.91871, + "18": 10.90007, + "19": 10.91327, + "20": 10.82088, + "21": 10.81366, + "22": 10.80501, + "23": 10.78962, + "24": 10.75486, + "25": 10.75928, + "26": 10.74825, + "27": 10.69711, + "28": 10.63743, + "29": 10.61526, + "30": 10.58797, + "31": 10.57883, + "32": 10.5583, + "33": 10.52369, + "34": 10.50303, "35": 10.4836, - "36": 10.46154, - "37": 10.45411, - "38": 10.45226, - "39": 10.40934, - "40": 10.40241, - "41": 10.37868, - "42": 10.32566, - "43": 10.3166, - "44": 10.27992, - "45": 10.30832, - "46": 10.2472, - "47": 10.23867, - "48": 10.22249, - "49": 10.20589, - "50": 10.18444, - "51": 10.19259, - "52": 10.15913, - "53": 10.16844, - "54": 10.11266, - "55": 10.08473, - "56": 10.10193, - "57": 10.1234, + "36": 10.4616, + "37": 10.4541, + "38": 10.45224, + "39": 10.40943, + "40": 10.40249, + "41": 10.37869, + "42": 10.32564, + "43": 10.31674, + "44": 10.27993, + "45": 10.30843, + "46": 10.24724, + "47": 10.23875, + "48": 10.22256, + "49": 10.20598, + "50": 10.1845, + "51": 10.19265, + "52": 10.15917, + "53": 10.1685, + "54": 10.11269, + "55": 10.08475, + "56": 10.10202, + "57": 10.12348, "58": 10.10239, - "59": 10.05154, - "60": 10.08328, - "61": 10.02215, - "62": 9.99326, - "63": 10.06757, - "64": 10.02651, - "65": 10.02787, - "66": 10.02544, - "67": 9.99694, - "68": 9.97259, - "69": 10.02185, - "70": 9.98222, - "71": 9.98457, - "72": 10.00073, - "73": 9.96104, - "74": 9.95153, - "75": 9.9143, - "76": 9.95246, - "77": 9.94443, - "78": 9.9106, - "79": 9.90321, - "80": 9.91084, - "81": 9.93261, - "82": 9.86692, - "83": 9.84165, - "84": 9.79596, - "85": 9.76662, - "86": 9.88041, - "87": 9.86768, - "88": 9.86223, - "89": 9.80258, - "90": 9.78297, - "91": 9.82699, - "92": 9.79113, - "93": 9.71999, - "94": 9.81833, - "95": 9.79024, - "96": 9.81142, - "97": 9.74197, - "98": 9.76131, - "99": 9.79043, - "100": 9.70982 + "59": 10.0516, + "60": 10.08336, + "61": 10.0222, + "62": 9.99328, + "63": 10.06761, + "64": 10.02659, + "65": 10.0279, + "66": 10.02551, + "67": 9.99697, + "68": 9.9726, + "69": 10.02186, + "70": 9.98233, + "71": 9.98462, + "72": 10.00081, + "73": 9.9611, + "74": 9.95154, + "75": 9.91439, + "76": 9.9525, + "77": 9.94444, + "78": 9.91065, + "79": 9.90324, + "80": 9.91086, + "81": 9.93267, + "82": 9.86695, + "83": 9.84171, + "84": 9.79603, + "85": 9.76671, + "86": 9.88046, + "87": 9.86776, + "88": 9.86231, + "89": 9.8027, + "90": 9.78301, + "91": 9.827, + "92": 9.79116, + "93": 9.72008, + "94": 9.81837, + "95": 9.7903, + "96": 9.81145, + "97": 9.74202, + "98": 9.76135, + "99": 9.79047, + "100": 9.70983 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22962222.0, - "2": 22849884.0, - "3": 22709196.0, - "4": 22778684.0, - "5": 22783510.0, - "6": 22746664.0, - "7": 22872212.0, - "8": 22611268.0, - "9": 22757986.0, - "10": 22489102.0, - "11": 22753568.0, - "12": 22640822.0, + "1": 22962356.0, + "2": 22849760.0, + "3": 22709136.0, + "4": 22778680.0, + "5": 22783668.0, + "6": 22746670.0, + "7": 22872268.0, + "8": 22611380.0, + "9": 22757932.0, + "10": 22489028.0, + "11": 22753676.0, + "12": 22640800.0, "13": 23322448.0, - "14": 22991720.0, - "15": 22717780.0, - "16": 22822716.0, - "17": 22929916.0, - "18": 22994544.0, - "19": 23084352.0, - "20": 22722842.0, - "21": 22919028.0, - "22": 22944564.0, - "23": 22628360.0, - "24": 22862176.0, - "25": 22636342.0, - "26": 23004756.0, - "27": 22802356.0, - "28": 22998960.0, - "29": 22979398.0, - "30": 22943856.0, - "31": 22907830.0, - "32": 22665828.0, - "33": 22740540.0, - "34": 23075828.0, - "35": 22751008.0, + "14": 22991658.0, + "15": 22717790.0, + "16": 22822774.0, + "17": 22929936.0, + "18": 22994672.0, + "19": 23084308.0, + "20": 22722826.0, + "21": 22918960.0, + "22": 22944572.0, + "23": 22628440.0, + "24": 22862080.0, + "25": 22636290.0, + "26": 23004680.0, + "27": 22802366.0, + "28": 22998918.0, + "29": 22979550.0, + "30": 22943904.0, + "31": 22907900.0, + "32": 22665848.0, + "33": 22740528.0, + "34": 23075928.0, + "35": 22751018.0, "36": 22695452.0, - "37": 23103496.0, - "38": 22963920.0, - "39": 22984932.0, - "40": 22749492.0, - "41": 23064274.0, - "42": 22688540.0, - "43": 22987284.0, - "44": 22705372.0, - "45": 22847172.0, - "46": 22733438.0, - "47": 22849270.0, - "48": 22833516.0, - "49": 22888964.0, - "50": 22644492.0, - "51": 22699114.0, - "52": 22813308.0, - "53": 22961874.0, - "54": 22786016.0, - "55": 22922864.0, - "56": 22660044.0, - "57": 23193390.0, - "58": 22689138.0, - "59": 22842072.0, - "60": 23013902.0, - "61": 22672740.0, - "62": 22732050.0, - "63": 22632330.0, - "64": 22998972.0, - "65": 23198328.0, - "66": 22694544.0, - "67": 22955768.0, - "68": 22924760.0, - "69": 23149964.0, - "70": 22821652.0, - "71": 22739272.0, - "72": 23116444.0, - "73": 23133048.0, - "74": 22941152.0, - "75": 22880638.0, - "76": 22702690.0, - "77": 22980352.0, - "78": 22977154.0, - "79": 22824012.0, - "80": 22931284.0, - "81": 22834754.0, - "82": 22730492.0, - "83": 22735268.0, - "84": 23099376.0, - "85": 22922288.0, - "86": 23068188.0, - "87": 22373858.0, - "88": 22555568.0, - "89": 22727848.0, - "90": 22766116.0, - "91": 22919320.0, - "92": 22672776.0, - "93": 22646124.0, - "94": 23128378.0, - "95": 22694986.0, - "96": 22844902.0, - "97": 22831948.0, - "98": 22875216.0, - "99": 22636834.0, - "100": 23000010.0 + "37": 23103392.0, + "38": 22963982.0, + "39": 22984994.0, + "40": 22749560.0, + "41": 23064300.0, + "42": 22688442.0, + "43": 22987388.0, + "44": 22705344.0, + "45": 22847152.0, + "46": 22733516.0, + "47": 22849348.0, + "48": 22833504.0, + "49": 22888996.0, + "50": 22644588.0, + "51": 22698992.0, + "52": 22813148.0, + "53": 22961906.0, + "54": 22786102.0, + "55": 22922978.0, + "56": 22659918.0, + "57": 23193362.0, + "58": 22689076.0, + "59": 22841974.0, + "60": 23013940.0, + "61": 22672824.0, + "62": 22731932.0, + "63": 22632314.0, + "64": 22998856.0, + "65": 23198306.0, + "66": 22694492.0, + "67": 22955878.0, + "68": 22924802.0, + "69": 23150054.0, + "70": 22821700.0, + "71": 22739400.0, + "72": 23116542.0, + "73": 23133058.0, + "74": 22941244.0, + "75": 22880492.0, + "76": 22702724.0, + "77": 22980612.0, + "78": 22977188.0, + "79": 22823984.0, + "80": 22931380.0, + "81": 22834816.0, + "82": 22730594.0, + "83": 22735228.0, + "84": 23099324.0, + "85": 22922364.0, + "86": 23068054.0, + "87": 22373852.0, + "88": 22555668.0, + "89": 22727868.0, + "90": 22766180.0, + "91": 22919328.0, + "92": 22672822.0, + "93": 22646240.0, + "94": 23128352.0, + "95": 22694976.0, + "96": 22845072.0, + "97": 22832008.0, + "98": 22875316.0, + "99": 22636810.0, + "100": 22999952.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.06947, - "3": 0.1325, - "4": 0.13148, - "5": 0.13399, - "6": 0.1307, - "7": 0.12914, - "8": 0.12961, - "9": 0.13237, - "10": 0.12969, - "11": 0.13307, - "12": 0.13058, - "13": 0.13359, - "14": 0.12993, - "15": 0.12945, - "16": 0.13047, - "17": 0.13018, - "18": 0.13138, - "19": 0.13287, - "20": 0.13272, - "21": 0.12973, - "22": 0.12966, - "23": 0.12885, - "24": 0.12915, - "25": 0.13099, - "26": 0.13074, - "27": 0.13092, - "28": 0.1292, - "29": 0.12943, - "30": 0.12915, - "31": 0.13064, - "32": 0.12875, - "33": 0.1288, - "34": 0.12897, - "35": 0.13008, - "36": 0.12938, - "37": 0.12887, - "38": 0.1292, - "39": 0.13043, - "40": 0.1299, - "41": 0.13058, - "42": 0.13287, - "43": 0.13083, - "44": 0.13041, - "45": 0.13029, - "46": 0.13071, - "47": 0.12945, - "48": 0.13094, - "49": 0.13019, - "50": 0.12821, - "51": 0.17187, - "52": 0.16994, - "53": 0.14122, - "54": 0.13376, - "55": 0.1306, - "56": 0.12956, - "57": 0.13098, - "58": 0.12962, - "59": 0.13213, - "60": 0.13079, - "61": 0.13249, - "62": 0.13214, - "63": 0.13059, - "64": 0.12978, - "65": 0.13175, - "66": 0.12867, - "67": 0.13339, - "68": 0.13442, - "69": 0.13091, - "70": 0.13231, - "71": 0.12906, - "72": 0.12935, - "73": 0.12996, - "74": 0.12895, - "75": 0.13049, - "76": 0.13039, - "77": 0.13206, - "78": 0.1302, - "79": 0.12975, - "80": 0.13035, - "81": 0.13017, - "82": 0.13307, - "83": 0.13048, - "84": 0.1336, - "85": 0.13085, - "86": 0.12931, - "87": 0.12852, - "88": 0.13013, - "89": 0.1294, - "90": 0.13005, - "91": 0.12987, - "92": 0.13082, - "93": 0.13156, - "94": 0.12954, - "95": 0.12818, - "96": 0.13009, - "97": 0.12906, - "98": 0.12838, - "99": 0.12875, - "100": 0.12869 + "2": 6.67982, + "3": 0.13539, + "4": 0.13689, + "5": 0.13411, + "6": 0.13296, + "7": 0.13372, + "8": 0.1373, + "9": 0.13611, + "10": 0.13387, + "11": 0.13761, + "12": 0.13338, + "13": 0.1337, + "14": 0.13418, + "15": 0.13316, + "16": 0.13488, + "17": 0.13342, + "18": 0.13311, + "19": 0.13534, + "20": 0.13371, + "21": 0.13353, + "22": 0.13397, + "23": 0.13355, + "24": 0.13362, + "25": 0.13464, + "26": 0.13489, + "27": 0.13505, + "28": 0.13379, + "29": 0.13469, + "30": 0.13305, + "31": 0.13343, + "32": 0.13483, + "33": 0.13521, + "34": 0.13384, + "35": 0.13321, + "36": 0.13309, + "37": 0.13288, + "38": 0.13249, + "39": 0.13571, + "40": 0.13409, + "41": 0.13404, + "42": 0.13708, + "43": 0.13789, + "44": 0.13361, + "45": 0.13824, + "46": 0.13434, + "47": 0.13344, + "48": 0.13357, + "49": 0.1337, + "50": 0.13445, + "51": 0.17717, + "52": 0.16454, + "53": 0.14147, + "54": 0.13672, + "55": 0.13721, + "56": 0.13635, + "57": 0.13262, + "58": 0.13369, + "59": 0.13329, + "60": 0.14116, + "61": 0.13667, + "62": 0.13539, + "63": 0.13393, + "64": 0.13403, + "65": 0.13432, + "66": 0.13402, + "67": 0.13459, + "68": 0.1333, + "69": 0.13356, + "70": 0.13553, + "71": 0.13401, + "72": 0.13481, + "73": 0.13461, + "74": 0.13565, + "75": 0.13437, + "76": 0.13538, + "77": 0.1342, + "78": 0.13409, + "79": 0.13526, + "80": 0.13444, + "81": 0.13821, + "82": 0.13145, + "83": 0.13487, + "84": 0.13592, + "85": 0.13438, + "86": 0.13461, + "87": 0.13511, + "88": 0.13704, + "89": 0.13673, + "90": 0.13449, + "91": 0.13422, + "92": 0.13588, + "93": 0.13246, + "94": 0.13308, + "95": 0.1339, + "96": 0.13218, + "97": 0.13347, + "98": 0.13261, + "99": 0.13393, + "100": 0.13288 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json index 6db0ba1d33f..28ca8c34a15 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94455, - "2": 10.94743, - "3": 10.94818, - "4": 10.94206, - "5": 10.95026, - "6": 10.94824, - "7": 10.93493, - "8": 10.94753, - "9": 10.93762, - "10": 10.94101, - "11": 10.93294, - "12": 10.94364, - "13": 10.90622, - "14": 10.9172, - "15": 10.89665, - "16": 10.89474, - "17": 10.88785, - "18": 10.86864, - "19": 10.87541, - "20": 10.81655, - "21": 10.78013, - "22": 10.77103, - "23": 10.76886, - "24": 10.74831, - "25": 10.74345, - "26": 10.72258, - "27": 10.6734, - "28": 10.60315, - "29": 10.57246, - "30": 10.56303, - "31": 10.54422, - "32": 10.53105, - "33": 10.48751, - "34": 10.47659, - "35": 10.45645, - "36": 10.44068, + "1": 10.94451, + "2": 10.94741, + "3": 10.94825, + "4": 10.94201, + "5": 10.95027, + "6": 10.94816, + "7": 10.93494, + "8": 10.94743, + "9": 10.93757, + "10": 10.94107, + "11": 10.93295, + "12": 10.94366, + "13": 10.90623, + "14": 10.91718, + "15": 10.89679, + "16": 10.89471, + "17": 10.88797, + "18": 10.86869, + "19": 10.8755, + "20": 10.81657, + "21": 10.78016, + "22": 10.77117, + "23": 10.76887, + "24": 10.74837, + "25": 10.74347, + "26": 10.72265, + "27": 10.67342, + "28": 10.60322, + "29": 10.57247, + "30": 10.56312, + "31": 10.54428, + "32": 10.53104, + "33": 10.48753, + "34": 10.47664, + "35": 10.45653, + "36": 10.44072, "37": 10.40765, - "38": 10.40511, - "39": 10.36532, - "40": 10.3672, - "41": 10.33527, - "42": 10.30512, - "43": 10.28352, - "44": 10.255, - "45": 10.2688, + "38": 10.40518, + "39": 10.3653, + "40": 10.3673, + "41": 10.33535, + "42": 10.30523, + "43": 10.28355, + "44": 10.25508, + "45": 10.26878, "46": 10.22196, - "47": 10.20823, - "48": 10.18878, - "49": 10.17404, - "50": 10.1681, - "51": 10.16394, - "52": 10.12751, - "53": 10.12716, - "54": 10.09558, - "55": 10.06465, - "56": 10.07494, - "57": 10.07258, - "58": 10.08235, - "59": 10.02218, - "60": 10.04248, - "61": 10.00366, - "62": 9.97281, - "63": 10.05249, - "64": 10.00735, - "65": 9.98001, + "47": 10.20826, + "48": 10.18885, + "49": 10.17403, + "50": 10.1682, + "51": 10.16398, + "52": 10.12763, + "53": 10.12719, + "54": 10.09562, + "55": 10.0647, + "56": 10.07497, + "57": 10.07271, + "58": 10.08236, + "59": 10.02224, + "60": 10.04256, + "61": 10.00376, + "62": 9.97284, + "63": 10.05248, + "64": 10.00736, + "65": 9.98009, "66": 9.99713, - "67": 9.96053, - "68": 9.93587, - "69": 9.9726, - "70": 9.94511, - "71": 9.96529, - "72": 9.94714, - "73": 9.92131, - "74": 9.91335, - "75": 9.88321, - "76": 9.92543, - "77": 9.90561, - "78": 9.86953, - "79": 9.86598, - "80": 9.87842, - "81": 9.89412, - "82": 9.82675, - "83": 9.80186, - "84": 9.74788, - "85": 9.72843, - "86": 9.84286, - "87": 9.85715, - "88": 9.83812, - "89": 9.77531, - "90": 9.76324, + "67": 9.96062, + "68": 9.93589, + "69": 9.97263, + "70": 9.94508, + "71": 9.96537, + "72": 9.94715, + "73": 9.92134, + "74": 9.91339, + "75": 9.8833, + "76": 9.92544, + "77": 9.90563, + "78": 9.86958, + "79": 9.86605, + "80": 9.87846, + "81": 9.89408, + "82": 9.82678, + "83": 9.80194, + "84": 9.74787, + "85": 9.72849, + "86": 9.84283, + "87": 9.85721, + "88": 9.83815, + "89": 9.77535, + "90": 9.76325, "91": 9.78325, - "92": 9.75571, - "93": 9.70188, - "94": 9.77557, - "95": 9.76597, - "96": 9.7691, - "97": 9.70025, - "98": 9.72317, - "99": 9.76326, - "100": 9.67042 + "92": 9.75581, + "93": 9.70192, + "94": 9.7756, + "95": 9.76601, + "96": 9.76916, + "97": 9.70033, + "98": 9.72326, + "99": 9.76327, + "100": 9.67049 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22961734.0, - "2": 22849292.0, - "3": 22709450.0, - "4": 22778272.0, - "5": 22783344.0, - "6": 22745688.0, - "7": 22871616.0, - "8": 22611788.0, - "9": 22758488.0, - "10": 22489664.0, - "11": 22753930.0, - "12": 22640498.0, - "13": 23322660.0, - "14": 22991940.0, - "15": 22718150.0, - "16": 22822544.0, - "17": 22929420.0, - "18": 22995424.0, + "1": 22961676.0, + "2": 22849324.0, + "3": 22709392.0, + "4": 22778208.0, + "5": 22783408.0, + "6": 22745682.0, + "7": 22871448.0, + "8": 22611742.0, + "9": 22758592.0, + "10": 22489788.0, + "11": 22754000.0, + "12": 22640580.0, + "13": 23322692.0, + "14": 22991980.0, + "15": 22718096.0, + "16": 22822408.0, + "17": 22929436.0, + "18": 22995432.0, "19": 23084828.0, - "20": 22723088.0, - "21": 22918220.0, - "22": 22945164.0, - "23": 22627484.0, - "24": 22862724.0, - "25": 22636220.0, - "26": 23004960.0, - "27": 22801468.0, - "28": 22999412.0, - "29": 22979382.0, - "30": 22943964.0, - "31": 22908184.0, - "32": 22665988.0, - "33": 22741296.0, - "34": 23076296.0, - "35": 22750520.0, - "36": 22695812.0, - "37": 23103140.0, - "38": 22964608.0, - "39": 22985704.0, - "40": 22749862.0, - "41": 23065512.0, - "42": 22687840.0, - "43": 22987484.0, - "44": 22704888.0, - "45": 22846928.0, - "46": 22734288.0, - "47": 22849852.0, - "48": 22833504.0, - "49": 22888668.0, - "50": 22644700.0, - "51": 22698624.0, - "52": 22812560.0, - "53": 22962504.0, - "54": 22786684.0, - "55": 22922392.0, - "56": 22660110.0, - "57": 23192904.0, - "58": 22688988.0, - "59": 22841898.0, - "60": 23013408.0, - "61": 22673272.0, - "62": 22732280.0, - "63": 22631808.0, - "64": 22999252.0, - "65": 23198302.0, - "66": 22694020.0, - "67": 22955048.0, - "68": 22924488.0, - "69": 23149376.0, - "70": 22822536.0, - "71": 22740440.0, - "72": 23117448.0, - "73": 23133530.0, - "74": 22941076.0, - "75": 22880362.0, - "76": 22702896.0, - "77": 22980008.0, - "78": 22977106.0, - "79": 22824536.0, - "80": 22930748.0, - "81": 22834604.0, - "82": 22731260.0, - "83": 22733720.0, - "84": 23099270.0, - "85": 22921714.0, - "86": 23069020.0, - "87": 22375336.0, - "88": 22555520.0, - "89": 22727918.0, - "90": 22765510.0, - "91": 22918544.0, - "92": 22672096.0, - "93": 22645440.0, - "94": 23128324.0, - "95": 22694938.0, - "96": 22845282.0, - "97": 22832220.0, - "98": 22874074.0, - "99": 22636662.0, - "100": 23000632.0 + "20": 22723064.0, + "21": 22918254.0, + "22": 22945122.0, + "23": 22627672.0, + "24": 22862716.0, + "25": 22636040.0, + "26": 23004826.0, + "27": 22801550.0, + "28": 22999472.0, + "29": 22979330.0, + "30": 22943888.0, + "31": 22908212.0, + "32": 22665978.0, + "33": 22741242.0, + "34": 23076366.0, + "35": 22750568.0, + "36": 22695876.0, + "37": 23103164.0, + "38": 22964606.0, + "39": 22985698.0, + "40": 22749752.0, + "41": 23065652.0, + "42": 22687862.0, + "43": 22987750.0, + "44": 22704962.0, + "45": 22846906.0, + "46": 22734100.0, + "47": 22849874.0, + "48": 22833476.0, + "49": 22888760.0, + "50": 22644576.0, + "51": 22698808.0, + "52": 22812564.0, + "53": 22962496.0, + "54": 22786708.0, + "55": 22922440.0, + "56": 22660224.0, + "57": 23192774.0, + "58": 22689076.0, + "59": 22841912.0, + "60": 23013384.0, + "61": 22673186.0, + "62": 22732284.0, + "63": 22631794.0, + "64": 22999306.0, + "65": 23198276.0, + "66": 22694010.0, + "67": 22955142.0, + "68": 22924460.0, + "69": 23149340.0, + "70": 22822656.0, + "71": 22740376.0, + "72": 23117308.0, + "73": 23133466.0, + "74": 22940986.0, + "75": 22880310.0, + "76": 22702712.0, + "77": 22979996.0, + "78": 22977068.0, + "79": 22824492.0, + "80": 22930828.0, + "81": 22834530.0, + "82": 22731320.0, + "83": 22733696.0, + "84": 23099368.0, + "85": 22921648.0, + "86": 23068952.0, + "87": 22375358.0, + "88": 22555496.0, + "89": 22727996.0, + "90": 22765896.0, + "91": 22918432.0, + "92": 22672088.0, + "93": 22645372.0, + "94": 23128346.0, + "95": 22694952.0, + "96": 22845202.0, + "97": 22832268.0, + "98": 22873974.0, + "99": 22636744.0, + "100": 23000626.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.15406, - "3": 2.8386, - "4": 2.51987, - "5": 2.04597, - "6": 2.28093, - "7": 1.89099, - "8": 2.46204, - "9": 2.52825, - "10": 2.19638, - "11": 2.69997, - "12": 1.90907, - "13": 2.47192, - "14": 2.66042, - "15": 2.13711, - "16": 2.21209, - "17": 2.92912, - "18": 2.52951, - "19": 2.23442, - "20": 2.20013, - "21": 2.52727, - "22": 1.85802, - "23": 2.38632, - "24": 2.23058, - "25": 2.59444, - "26": 2.57789, - "27": 2.27719, - "28": 2.34143, - "29": 1.92447, - "30": 1.99388, - "31": 1.73962, - "32": 3.35076, - "33": 1.82019, - "34": 1.91928, - "35": 2.69899, - "36": 2.49125, - "37": 2.29016, - "38": 1.65719, - "39": 2.95518, - "40": 2.55015, - "41": 2.13436, - "42": 3.33939, - "43": 2.54565, - "44": 3.25779, - "45": 2.60464, - "46": 2.12982, - "47": 2.88836, - "48": 2.64484, - "49": 2.10153, - "50": 2.56157, - "51": 1.96443, - "52": 2.49422, - "53": 1.91775, - "54": 2.69343, - "55": 2.5621, - "56": 2.81395, - "57": 1.77504, - "58": 2.57933, - "59": 2.17979, - "60": 2.18026, - "61": 2.83063, - "62": 3.02627, - "63": 2.08661, - "64": 2.23306, - "65": 2.21281, - "66": 2.79388, - "67": 2.24457, - "68": 1.82545, - "69": 2.36781, - "70": 2.3322, - "71": 2.63242, - "72": 2.20177, - "73": 2.3826, - "74": 2.17222, - "75": 2.19764, - "76": 2.33719, - "77": 2.17667, - "78": 2.15935, - "79": 2.67024, - "80": 2.04535, - "81": 2.45699, - "82": 2.5877, - "83": 2.1103, - "84": 2.31754, - "85": 3.84649, - "86": 2.31638, - "87": 2.86869, - "88": 2.90248, - "89": 3.28206, - "90": 2.45151, - "91": 2.54808, - "92": 3.15338, - "93": 2.26075, - "94": 2.85946, - "95": 2.7598, - "96": 2.08416, - "97": 2.13821, - "98": 2.27557, - "99": 2.67809, - "100": 2.27389 + "2": 6.21756, + "3": 1.79936, + "4": 1.66478, + "5": 1.48217, + "6": 1.55288, + "7": 1.7125, + "8": 2.1843, + "9": 2.37634, + "10": 1.95936, + "11": 1.90331, + "12": 1.66364, + "13": 2.06992, + "14": 2.11736, + "15": 1.72268, + "16": 1.82524, + "17": 2.46437, + "18": 1.78126, + "19": 1.87934, + "20": 1.75864, + "21": 2.17251, + "22": 1.46617, + "23": 1.76141, + "24": 1.33376, + "25": 2.12627, + "26": 2.03802, + "27": 1.73671, + "28": 1.77058, + "29": 1.35331, + "30": 1.55353, + "31": 1.64716, + "32": 2.71949, + "33": 1.51524, + "34": 1.46793, + "35": 2.10103, + "36": 1.9853, + "37": 1.92595, + "38": 1.61153, + "39": 2.16743, + "40": 1.90755, + "41": 2.0328, + "42": 2.37963, + "43": 2.02858, + "44": 1.5203, + "45": 2.13588, + "46": 1.75068, + "47": 1.97068, + "48": 2.23887, + "49": 1.91589, + "50": 2.33417, + "51": 0.92061, + "52": 2.45315, + "53": 1.78576, + "54": 2.04965, + "55": 1.87342, + "56": 2.03176, + "57": 1.14495, + "58": 1.91952, + "59": 1.33553, + "60": 1.21917, + "61": 1.5207, + "62": 1.87159, + "63": 1.30126, + "64": 1.60787, + "65": 1.53789, + "66": 1.91339, + "67": 1.70281, + "68": 1.13818, + "69": 1.44205, + "70": 1.70089, + "71": 1.82927, + "72": 1.5869, + "73": 1.35596, + "74": 1.33961, + "75": 1.57728, + "76": 1.80038, + "77": 1.63489, + "78": 1.49197, + "79": 2.14753, + "80": 1.47431, + "81": 1.78371, + "82": 2.02726, + "83": 1.77973, + "84": 1.2747, + "85": 1.7825, + "86": 1.80697, + "87": 2.46629, + "88": 1.83694, + "89": 2.00987, + "90": 1.73109, + "91": 2.04594, + "92": 2.14384, + "93": 1.41638, + "94": 1.66148, + "95": 2.53773, + "96": 1.98614, + "97": 1.5099, + "98": 1.9077, + "99": 2.70274, + "100": 1.85471 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json index f8d630aa812..5cb2dced98b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.96459, - "2": 10.95289, + "1": 10.96455, + "2": 10.95282, "3": 10.95803, - "4": 10.95244, - "5": 10.95248, - "6": 10.95358, - "7": 10.95137, - "8": 10.95114, - "9": 10.95832, - "10": 10.94456, - "11": 10.93658, - "12": 10.94322, - "13": 10.94355, - "14": 10.94301, - "15": 10.91488, - "16": 10.90027, - "17": 10.91041, - "18": 10.90618, - "19": 10.90182, - "20": 10.80739, + "4": 10.95242, + "5": 10.95253, + "6": 10.95361, + "7": 10.95139, + "8": 10.95109, + "9": 10.9583, + "10": 10.94445, + "11": 10.93655, + "12": 10.94323, + "13": 10.94349, + "14": 10.94298, + "15": 10.9148, + "16": 10.90029, + "17": 10.91038, + "18": 10.9061, + "19": 10.90183, + "20": 10.80731, "21": 10.79426, - "22": 10.80475, - "23": 10.78953, - "24": 10.77488, - "25": 10.76205, - "26": 10.75482, - "27": 10.71916, + "22": 10.80476, + "23": 10.78952, + "24": 10.7749, + "25": 10.76207, + "26": 10.75487, + "27": 10.7191, "28": 10.63825, - "29": 10.60841, - "30": 10.5862, - "31": 10.59086, + "29": 10.60844, + "30": 10.58628, + "31": 10.59085, "32": 10.57067, - "33": 10.53985, + "33": 10.5399, "34": 10.49922, "35": 10.50003, - "36": 10.48548, - "37": 10.44895, - "38": 10.45016, - "39": 10.41721, - "40": 10.39508, - "41": 10.37463, - "42": 10.36017, - "43": 10.33036, - "44": 10.31849, - "45": 10.31555, - "46": 10.27647, - "47": 10.26789, - "48": 10.21799, - "49": 10.2135, - "50": 10.22186 + "36": 10.48545, + "37": 10.449, + "38": 10.45019, + "39": 10.41717, + "40": 10.39513, + "41": 10.37477, + "42": 10.36024, + "43": 10.33042, + "44": 10.31854, + "45": 10.31567, + "46": 10.27655, + "47": 10.26794, + "48": 10.218, + "49": 10.21352, + "50": 10.22191 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1870.0, - "2": 1721.0, - "3": 1853.0, - "4": 1776.0, - "5": 1802.0, - "6": 1726.0, - "7": 1933.0, - "8": 1721.0, - "9": 1713.0, - "10": 1749.0, - "11": 1674.0, - "12": 1715.0, - "13": 1859.0, - "14": 1838.0, - "15": 1604.0, - "16": 1770.0, - "17": 1822.0, - "18": 1841.0, - "19": 1648.0, - "20": 1756.0, - "21": 1757.0, - "22": 1731.0, - "23": 1766.0, - "24": 1747.0, - "25": 1668.0, - "26": 1803.0, - "27": 1821.0, - "28": 1767.0, - "29": 1855.0, - "30": 1754.0, - "31": 1997.0, - "32": 1955.0, - "33": 1901.0, - "34": 1959.0, - "35": 2073.0, - "36": 1993.0, - "37": 2152.0, - "38": 2056.0, - "39": 2209.0, - "40": 2206.0, - "41": 2226.0, - "42": 2022.0, - "43": 2364.0, - "44": 2204.0, - "45": 2468.0, - "46": 2353.0, - "47": 2380.0, - "48": 2508.0, - "49": 2742.0, - "50": 2494.0 + "1": 1770.0, + "2": 1689.0, + "3": 1797.0, + "4": 1700.0, + "5": 1742.0, + "6": 1764.0, + "7": 1919.0, + "8": 1639.0, + "9": 1699.0, + "10": 1768.0, + "11": 1700.0, + "12": 1684.0, + "13": 1854.0, + "14": 1902.0, + "15": 1571.0, + "16": 1714.0, + "17": 1833.0, + "18": 1822.0, + "19": 1617.0, + "20": 1646.0, + "21": 1886.0, + "22": 1768.0, + "23": 1768.0, + "24": 1731.0, + "25": 1700.0, + "26": 1793.0, + "27": 1755.0, + "28": 1799.0, + "29": 1894.0, + "30": 1820.0, + "31": 2013.0, + "32": 1935.0, + "33": 1968.0, + "34": 1991.0, + "35": 2074.0, + "36": 1985.0, + "37": 2199.0, + "38": 2055.0, + "39": 2176.0, + "40": 2288.0, + "41": 2224.0, + "42": 2000.0, + "43": 2326.0, + "44": 2167.0, + "45": 2443.0, + "46": 2432.0, + "47": 2371.0, + "48": 2599.0, + "49": 2687.0, + "50": 2576.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.02254, - "3": 0.12256, - "4": 0.124, - "5": 0.12217, - "6": 0.12111, - "7": 0.12333, - "8": 0.12063, - "9": 0.12652, - "10": 0.12735, - "11": 0.12426, - "12": 0.12945, - "13": 0.12193, - "14": 0.11999, - "15": 0.11908, - "16": 0.11846, - "17": 0.1209, - "18": 0.12537, - "19": 0.12222, - "20": 0.12047, - "21": 0.12121, - "22": 0.12216, - "23": 0.12369, - "24": 0.12274, - "25": 0.1184, - "26": 0.1209, - "27": 0.12289, - "28": 0.12143, - "29": 0.12385, - "30": 0.12146, - "31": 0.12197, - "32": 0.12352, - "33": 0.12352, - "34": 0.12118, - "35": 0.12124, - "36": 0.12427, - "37": 0.12542, - "38": 0.1255, - "39": 0.12545, - "40": 0.11878, - "41": 0.12106, - "42": 0.12173, - "43": 0.12535, - "44": 0.12147, + "2": 6.76559, + "3": 0.1235, + "4": 0.12536, + "5": 0.12275, + "6": 0.12292, + "7": 0.12239, + "8": 0.11935, + "9": 0.12123, + "10": 0.12065, + "11": 0.12219, + "12": 0.11957, + "13": 0.12062, + "14": 0.11936, + "15": 0.1221, + "16": 0.12288, + "17": 0.12518, + "18": 0.1262, + "19": 0.12263, + "20": 0.12728, + "21": 0.12251, + "22": 0.12321, + "23": 0.12864, + "24": 0.1247, + "25": 0.12484, + "26": 0.12432, + "27": 0.12801, + "28": 0.12606, + "29": 0.12617, + "30": 0.12574, + "31": 0.12264, + "32": 0.12741, + "33": 0.12556, + "34": 0.12572, + "35": 0.12416, + "36": 0.1267, + "37": 0.12517, + "38": 0.12437, + "39": 0.12612, + "40": 0.11997, + "41": 0.12256, + "42": 0.12567, + "43": 0.12476, + "44": 0.12368, "45": 0.12391, - "46": 0.12543, - "47": 0.12069, - "48": 0.12417, - "49": 0.12556, - "50": 0.12213 + "46": 0.12533, + "47": 0.12414, + "48": 0.12628, + "49": 0.12358, + "50": 0.12497 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json index cccd6bf832b..6a5ae22b49e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93805, - "2": 10.93722, - "3": 10.93622, + "1": 10.93802, + "2": 10.93731, + "3": 10.93625, "4": 10.94202, - "5": 10.93341, - "6": 10.93239, - "7": 10.9378, - "8": 10.93336, - "9": 10.93523, - "10": 10.93708, - "11": 10.92541, - "12": 10.92478, - "13": 10.92498, - "14": 10.913, - "15": 10.89303, - "16": 10.88848, - "17": 10.89973, - "18": 10.87744, - "19": 10.87629, - "20": 10.79078, - "21": 10.78028, - "22": 10.77488, - "23": 10.77493, + "5": 10.93343, + "6": 10.93236, + "7": 10.93773, + "8": 10.93332, + "9": 10.93526, + "10": 10.93703, + "11": 10.92535, + "12": 10.92481, + "13": 10.92504, + "14": 10.91293, + "15": 10.89304, + "16": 10.88857, + "17": 10.89967, + "18": 10.87741, + "19": 10.87633, + "20": 10.79086, + "21": 10.78034, + "22": 10.77493, + "23": 10.77501, "24": 10.7363, - "25": 10.74377, + "25": 10.74381, "26": 10.72424, - "27": 10.69604, - "28": 10.62337, - "29": 10.6032, - "30": 10.58016, - "31": 10.56927, - "32": 10.54831, - "33": 10.51986, - "34": 10.48554, - "35": 10.49212, - "36": 10.47099, - "37": 10.43652, - "38": 10.44075, - "39": 10.40244, - "40": 10.39244, - "41": 10.3684, - "42": 10.34588, - "43": 10.32635, - "44": 10.28947, - "45": 10.30802, - "46": 10.26791, - "47": 10.25248, - "48": 10.20359, - "49": 10.2011, - "50": 10.20928 + "27": 10.69605, + "28": 10.62339, + "29": 10.60317, + "30": 10.58018, + "31": 10.5693, + "32": 10.54838, + "33": 10.51995, + "34": 10.48559, + "35": 10.4921, + "36": 10.47113, + "37": 10.43649, + "38": 10.44083, + "39": 10.40248, + "40": 10.39252, + "41": 10.36846, + "42": 10.34595, + "43": 10.32632, + "44": 10.28954, + "45": 10.30813, + "46": 10.268, + "47": 10.25254, + "48": 10.2036, + "49": 10.20116, + "50": 10.2093 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1793.0, - "2": 1843.0, - "3": 1760.0, - "4": 1725.0, - "5": 1766.0, - "6": 1737.0, - "7": 2008.0, - "8": 1734.0, - "9": 1810.0, - "10": 1808.0, - "11": 1609.0, - "12": 1629.0, - "13": 1860.0, - "14": 1810.0, - "15": 1650.0, - "16": 1834.0, - "17": 1793.0, - "18": 1784.0, - "19": 1710.0, - "20": 1737.0, - "21": 1773.0, - "22": 1764.0, - "23": 1702.0, - "24": 1899.0, - "25": 1744.0, - "26": 1836.0, - "27": 1790.0, - "28": 1869.0, - "29": 1952.0, - "30": 1853.0, - "31": 1955.0, - "32": 1968.0, - "33": 2035.0, - "34": 2015.0, - "35": 2058.0, - "36": 1929.0, - "37": 2202.0, - "38": 2144.0, - "39": 2101.0, - "40": 2199.0, - "41": 2343.0, - "42": 1977.0, - "43": 2341.0, - "44": 2128.0, - "45": 2447.0, - "46": 2354.0, - "47": 2413.0, - "48": 2564.0, - "49": 2774.0, - "50": 2598.0 + "1": 1847.0, + "2": 1732.0, + "3": 1830.0, + "4": 1772.0, + "5": 1804.0, + "6": 1733.0, + "7": 1999.0, + "8": 1621.0, + "9": 1833.0, + "10": 1782.0, + "11": 1707.0, + "12": 1783.0, + "13": 1896.0, + "14": 1815.0, + "15": 1743.0, + "16": 1712.0, + "17": 1797.0, + "18": 1865.0, + "19": 1728.0, + "20": 1733.0, + "21": 1827.0, + "22": 1744.0, + "23": 1595.0, + "24": 1910.0, + "25": 1714.0, + "26": 1789.0, + "27": 1814.0, + "28": 1818.0, + "29": 1976.0, + "30": 1876.0, + "31": 2034.0, + "32": 2040.0, + "33": 2031.0, + "34": 2007.0, + "35": 2082.0, + "36": 2001.0, + "37": 2176.0, + "38": 2130.0, + "39": 2111.0, + "40": 2350.0, + "41": 2375.0, + "42": 2005.0, + "43": 2345.0, + "44": 2223.0, + "45": 2457.0, + "46": 2379.0, + "47": 2442.0, + "48": 2573.0, + "49": 2799.0, + "50": 2611.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.96224, - "3": 2.3218, - "4": 2.17825, - "5": 1.70713, - "6": 1.96774, - "7": 1.53587, - "8": 2.45493, - "9": 2.97904, - "10": 2.29226, - "11": 2.69633, - "12": 2.13665, - "13": 2.50903, - "14": 1.89119, - "15": 2.14381, - "16": 2.00981, - "17": 2.91275, - "18": 2.36625, - "19": 1.95486, - "20": 1.67776, - "21": 2.00545, - "22": 1.32206, - "23": 2.11073, - "24": 1.80439, - "25": 2.60464, - "26": 2.20412, - "27": 1.87553, - "28": 2.02727, - "29": 1.56077, - "30": 1.56044, - "31": 1.52813, - "32": 2.45511, - "33": 1.44263, - "34": 1.61757, - "35": 2.05391, - "36": 2.0777, - "37": 1.84558, - "38": 1.6175, - "39": 2.18046, - "40": 1.51716, - "41": 1.96475, - "42": 2.54763, - "43": 1.81373, - "44": 1.64848, - "45": 1.99012, - "46": 1.71886, - "47": 1.7574, - "48": 2.11413, - "49": 1.46823, - "50": 2.03845 + "2": 6.38734, + "3": 1.93371, + "4": 1.46641, + "5": 1.31973, + "6": 1.64138, + "7": 1.07997, + "8": 1.96412, + "9": 2.02282, + "10": 1.80547, + "11": 1.87937, + "12": 1.58986, + "13": 1.83797, + "14": 1.42969, + "15": 1.53769, + "16": 1.80613, + "17": 2.33338, + "18": 1.64231, + "19": 1.70785, + "20": 1.78986, + "21": 2.00339, + "22": 1.49156, + "23": 1.8608, + "24": 1.36903, + "25": 2.11624, + "26": 2.15353, + "27": 1.65163, + "28": 1.81226, + "29": 1.47795, + "30": 1.43772, + "31": 1.25869, + "32": 2.08825, + "33": 1.32136, + "34": 1.51988, + "35": 1.70336, + "36": 1.79312, + "37": 1.60581, + "38": 1.4398, + "39": 1.87911, + "40": 1.37871, + "41": 1.53444, + "42": 2.37669, + "43": 1.73056, + "44": 1.4992, + "45": 2.0894, + "46": 1.62847, + "47": 1.87022, + "48": 1.97808, + "49": 1.33568, + "50": 1.55335 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json index 456fd0e7b9b..48a0b817282 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.96459, - "2": 10.95289, + "1": 10.96455, + "2": 10.95282, "3": 10.95803, - "4": 10.95244, - "5": 10.95248, - "6": 10.95358, - "7": 10.95137, - "8": 10.95114, - "9": 10.95832, - "10": 10.94456, - "11": 10.93658, - "12": 10.94324, - "13": 10.9435, - "14": 10.94301, - "15": 10.91487, - "16": 10.90024, - "17": 10.91038, - "18": 10.9062, + "4": 10.95242, + "5": 10.95253, + "6": 10.95361, + "7": 10.95139, + "8": 10.95109, + "9": 10.9583, + "10": 10.94445, + "11": 10.93655, + "12": 10.94322, + "13": 10.94347, + "14": 10.943, + "15": 10.91485, + "16": 10.90026, + "17": 10.91034, + "18": 10.90611, "19": 10.90187, - "20": 10.80737, - "21": 10.79428, - "22": 10.80475, - "23": 10.78952, - "24": 10.7749, - "25": 10.76208, - "26": 10.75479, + "20": 10.80734, + "21": 10.79429, + "22": 10.80478, + "23": 10.78955, + "24": 10.77489, + "25": 10.76205, + "26": 10.75484, "27": 10.71913, - "28": 10.63827, - "29": 10.60838, - "30": 10.58622, - "31": 10.59086, - "32": 10.57066, - "33": 10.53982, - "34": 10.4992, - "35": 10.50001, - "36": 10.48547, + "28": 10.63824, + "29": 10.60843, + "30": 10.58626, + "31": 10.59084, + "32": 10.5707, + "33": 10.53991, + "34": 10.49925, + "35": 10.50003, + "36": 10.48545, "37": 10.44898, - "38": 10.45018, - "39": 10.41717, - "40": 10.39509, - "41": 10.37465, - "42": 10.36016, - "43": 10.33036, - "44": 10.31849, - "45": 10.31556, - "46": 10.27648, - "47": 10.2679, - "48": 10.21803, - "49": 10.21349, - "50": 10.22186 + "38": 10.45024, + "39": 10.41718, + "40": 10.39512, + "41": 10.37475, + "42": 10.36025, + "43": 10.33043, + "44": 10.31856, + "45": 10.31564, + "46": 10.27655, + "47": 10.26796, + "48": 10.21798, + "49": 10.21355, + "50": 10.22191 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1870.0, - "2": 1721.0, - "3": 1853.0, - "4": 1776.0, - "5": 1802.0, - "6": 1726.0, - "7": 1933.0, - "8": 1749.0, - "9": 1799.0, - "10": 1776.0, - "11": 1689.0, - "12": 1714.0, - "13": 1782.0, - "14": 1911.0, - "15": 1584.0, - "16": 1800.0, - "17": 1882.0, - "18": 1841.0, - "19": 1641.0, - "20": 1762.0, - "21": 1886.0, - "22": 1745.0, - "23": 1791.0, - "24": 1806.0, - "25": 1719.0, - "26": 1725.0, - "27": 1790.0, - "28": 1672.0, - "29": 1907.0, - "30": 1768.0, - "31": 2048.0, - "32": 1910.0, - "33": 1954.0, - "34": 2036.0, - "35": 2084.0, - "36": 2015.0, - "37": 2210.0, - "38": 2054.0, - "39": 2199.0, - "40": 2183.0, - "41": 2318.0, - "42": 1917.0, - "43": 2402.0, - "44": 2091.0, - "45": 2433.0, - "46": 2384.0, - "47": 2340.0, - "48": 2594.0, - "49": 2773.0, - "50": 2561.0 + "1": 1770.0, + "2": 1689.0, + "3": 1797.0, + "4": 1700.0, + "5": 1742.0, + "6": 1764.0, + "7": 1919.0, + "8": 1639.0, + "9": 1699.0, + "10": 1768.0, + "11": 1719.0, + "12": 1710.0, + "13": 1860.0, + "14": 1850.0, + "15": 1697.0, + "16": 1768.0, + "17": 1884.0, + "18": 1779.0, + "19": 1665.0, + "20": 1773.0, + "21": 1693.0, + "22": 1695.0, + "23": 1735.0, + "24": 1775.0, + "25": 1597.0, + "26": 1762.0, + "27": 1849.0, + "28": 1723.0, + "29": 1983.0, + "30": 1775.0, + "31": 2077.0, + "32": 2029.0, + "33": 1981.0, + "34": 2022.0, + "35": 1970.0, + "36": 2026.0, + "37": 2204.0, + "38": 2088.0, + "39": 2220.0, + "40": 2225.0, + "41": 2278.0, + "42": 1977.0, + "43": 2394.0, + "44": 2109.0, + "45": 2455.0, + "46": 2344.0, + "47": 2417.0, + "48": 2602.0, + "49": 2782.0, + "50": 2527.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.88607, - "3": 0.12398, - "4": 0.12548, - "5": 0.12607, - "6": 0.12929, - "7": 0.12376, - "8": 0.12665, - "9": 0.12501, - "10": 0.12664, - "11": 0.35519, - "12": 0.12845, - "13": 0.12535, - "14": 0.12538, - "15": 0.12546, - "16": 0.1268, - "17": 0.12657, - "18": 0.12789, - "19": 0.12916, - "20": 0.12759, - "21": 0.35106, - "22": 0.13041, - "23": 0.13084, - "24": 0.1295, - "25": 0.12814, - "26": 0.12883, - "27": 0.13337, - "28": 0.13301, - "29": 0.13306, - "30": 0.13191, - "31": 0.35306, - "32": 0.12533, - "33": 0.12783, - "34": 0.12786, - "35": 0.12871, - "36": 0.12987, - "37": 0.12593, - "38": 0.12987, - "39": 0.12794, - "40": 0.12959, - "41": 0.35629, - "42": 0.12969, - "43": 0.12633, - "44": 0.13152, - "45": 0.13006, - "46": 0.12691, - "47": 0.12701, - "48": 0.13036, - "49": 0.13103, - "50": 0.12901 + "2": 7.68801, + "3": 0.13101, + "4": 0.13939, + "5": 0.13572, + "6": 0.13279, + "7": 0.13495, + "8": 0.13269, + "9": 0.13139, + "10": 0.13043, + "11": 0.3654, + "12": 0.13266, + "13": 0.13192, + "14": 0.12966, + "15": 0.13171, + "16": 0.13141, + "17": 0.14318, + "18": 0.1369, + "19": 0.13478, + "20": 0.14158, + "21": 0.36834, + "22": 0.13547, + "23": 0.14079, + "24": 0.14283, + "25": 0.13893, + "26": 0.14247, + "27": 0.14699, + "28": 0.14065, + "29": 0.13761, + "30": 0.13639, + "31": 0.36477, + "32": 0.14472, + "33": 0.13058, + "34": 0.13302, + "35": 0.1274, + "36": 0.13343, + "37": 0.13381, + "38": 0.13276, + "39": 0.131, + "40": 0.12695, + "41": 0.36091, + "42": 0.13081, + "43": 0.1317, + "44": 0.13473, + "45": 0.13729, + "46": 0.13019, + "47": 0.13164, + "48": 0.13562, + "49": 0.13384, + "50": 0.13037 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json index e3bb29cc9bb..bf506807d5a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93805, - "2": 10.93722, - "3": 10.93622, + "1": 10.93802, + "2": 10.93731, + "3": 10.93625, "4": 10.94202, - "5": 10.93341, - "6": 10.93239, - "7": 10.9378, - "8": 10.93336, - "9": 10.93523, - "10": 10.93708, - "11": 10.92541, + "5": 10.93343, + "6": 10.93236, + "7": 10.93773, + "8": 10.93332, + "9": 10.93526, + "10": 10.93703, + "11": 10.92537, "12": 10.92478, - "13": 10.92498, - "14": 10.913, - "15": 10.89303, - "16": 10.88848, - "17": 10.89973, - "18": 10.87744, - "19": 10.8763, - "20": 10.79072, + "13": 10.925, + "14": 10.91289, + "15": 10.89306, + "16": 10.88857, + "17": 10.89968, + "18": 10.87739, + "19": 10.87631, + "20": 10.79086, "21": 10.78034, - "22": 10.77489, - "23": 10.77493, - "24": 10.73628, - "25": 10.74375, - "26": 10.72423, - "27": 10.69602, - "28": 10.62335, - "29": 10.60318, - "30": 10.58016, - "31": 10.56934, - "32": 10.54832, - "33": 10.51985, - "34": 10.48553, - "35": 10.4921, - "36": 10.47101, - "37": 10.43653, - "38": 10.44075, - "39": 10.40247, - "40": 10.39246, - "41": 10.36838, - "42": 10.34589, - "43": 10.32638, - "44": 10.28949, - "45": 10.30803, - "46": 10.26792, - "47": 10.25248, - "48": 10.20356, - "49": 10.20108, - "50": 10.20926 + "22": 10.77492, + "23": 10.77501, + "24": 10.73633, + "25": 10.74379, + "26": 10.72426, + "27": 10.69605, + "28": 10.62339, + "29": 10.60324, + "30": 10.58015, + "31": 10.56932, + "32": 10.5484, + "33": 10.51996, + "34": 10.48559, + "35": 10.49212, + "36": 10.47112, + "37": 10.43649, + "38": 10.44078, + "39": 10.40249, + "40": 10.3925, + "41": 10.36846, + "42": 10.34595, + "43": 10.32635, + "44": 10.28952, + "45": 10.30812, + "46": 10.26798, + "47": 10.25255, + "48": 10.20361, + "49": 10.20112, + "50": 10.20931 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1793.0, - "2": 1843.0, - "3": 1760.0, - "4": 1725.0, - "5": 1766.0, - "6": 1737.0, - "7": 2008.0, - "8": 1734.0, - "9": 1810.0, - "10": 1808.0, - "11": 1609.0, - "12": 1629.0, - "13": 1860.0, - "14": 1810.0, - "15": 1650.0, - "16": 1834.0, - "17": 1793.0, - "18": 1785.0, - "19": 1678.0, - "20": 1648.0, - "21": 1762.0, - "22": 1718.0, - "23": 1666.0, - "24": 1852.0, - "25": 1701.0, - "26": 1848.0, - "27": 1834.0, - "28": 1871.0, - "29": 2032.0, - "30": 1892.0, - "31": 2038.0, - "32": 1994.0, - "33": 2040.0, - "34": 2067.0, - "35": 2022.0, - "36": 1876.0, + "1": 1847.0, + "2": 1732.0, + "3": 1830.0, + "4": 1772.0, + "5": 1804.0, + "6": 1733.0, + "7": 1999.0, + "8": 1621.0, + "9": 1833.0, + "10": 1782.0, + "11": 1669.0, + "12": 1672.0, + "13": 1869.0, + "14": 1823.0, + "15": 1676.0, + "16": 1761.0, + "17": 1797.0, + "18": 1883.0, + "19": 1643.0, + "20": 1736.0, + "21": 1802.0, + "22": 1713.0, + "23": 1674.0, + "24": 1885.0, + "25": 1673.0, + "26": 1912.0, + "27": 1877.0, + "28": 1921.0, + "29": 1967.0, + "30": 1857.0, + "31": 1996.0, + "32": 2028.0, + "33": 1988.0, + "34": 2080.0, + "35": 2049.0, + "36": 1936.0, "37": 2201.0, - "38": 2209.0, - "39": 2128.0, - "40": 2321.0, - "41": 2376.0, - "42": 1900.0, - "43": 2332.0, - "44": 2199.0, - "45": 2583.0, - "46": 2490.0, - "47": 2430.0, - "48": 2588.0, - "49": 2814.0, - "50": 2547.0 + "38": 2121.0, + "39": 2141.0, + "40": 2336.0, + "41": 2327.0, + "42": 1891.0, + "43": 2274.0, + "44": 2249.0, + "45": 2399.0, + "46": 2389.0, + "47": 2451.0, + "48": 2595.0, + "49": 2728.0, + "50": 2500.0 } }, "mem-allocated-bytes": { @@ -134,7 +134,7 @@ "14": 761634304.0, "15": 761634304.0, "16": 761634304.0, - "17": 761634304.0, + "17": 762682880.0, "18": 761634304.0, "19": 761634304.0, "20": 761634304.0, @@ -150,7 +150,7 @@ "30": 761634304.0, "31": 761634304.0, "32": 761634304.0, - "33": 762158592.0, + "33": 761634304.0, "34": 761634304.0, "35": 761634304.0, "36": 761634304.0, @@ -176,17 +176,17 @@ "step_interval": 1, "values": { "1": 3868768768.0, - "2": 4153623552.0, - "3": 4153623552.0, - "4": 4153623552.0, - "5": 4153623552.0, - "6": 4153623552.0, - "7": 4153623552.0, - "8": 4153623552.0, - "9": 4153623552.0, - "10": 4153623552.0, - "11": 4153623552.0, - "12": 4153623552.0, + "2": 4152574464.0, + "3": 4152574464.0, + "4": 4152574464.0, + "5": 4152574464.0, + "6": 4152574464.0, + "7": 4152574464.0, + "8": 4153361408.0, + "9": 4153361408.0, + "10": 4153361408.0, + "11": 4153361408.0, + "12": 4153361408.0, "13": 4153623552.0, "14": 4153623552.0, "15": 4153623552.0, @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.67523, - "3": 2.27552, - "4": 1.79312, - "5": 1.72626, - "6": 1.68961, - "7": 1.90367, - "8": 2.16647, - "9": 1.81026, - "10": 2.21507, - "11": 2.34321, - "12": 1.7015, - "13": 1.82959, - "14": 2.34278, - "15": 1.58959, - "16": 1.801, - "17": 2.13046, - "18": 1.78159, - "19": 2.03672, - "20": 1.64137, - "21": 1.97082, - "22": 1.6638, - "23": 2.23761, - "24": 1.80264, - "25": 2.31038, - "26": 2.42274, - "27": 1.82868, - "28": 1.91514, - "29": 1.58088, - "30": 1.9589, - "31": 1.75576, - "32": 3.03192, - "33": 1.51195, - "34": 1.58649, - "35": 2.12525, - "36": 2.22916, - "37": 2.02127, - "38": 1.51169, - "39": 2.3559, - "40": 2.17475, - "41": 2.11859, - "42": 2.73378, - "43": 1.74168, - "44": 1.8561, - "45": 2.44526, - "46": 1.83278, - "47": 1.88328, - "48": 2.2646, - "49": 1.79655, - "50": 2.10757 + "2": 7.1282, + "3": 1.92486, + "4": 1.80452, + "5": 1.0832, + "6": 1.85874, + "7": 1.20029, + "8": 1.75598, + "9": 1.78314, + "10": 1.44508, + "11": 1.65646, + "12": 1.53143, + "13": 1.54526, + "14": 1.60355, + "15": 1.51974, + "16": 1.62581, + "17": 1.65544, + "18": 1.59234, + "19": 1.65399, + "20": 1.41259, + "21": 1.78764, + "22": 1.24162, + "23": 1.72932, + "24": 1.29872, + "25": 1.65175, + "26": 2.10177, + "27": 1.78179, + "28": 1.46621, + "29": 1.74055, + "30": 1.58621, + "31": 1.18393, + "32": 2.69066, + "33": 1.34016, + "34": 1.34076, + "35": 1.74385, + "36": 1.88453, + "37": 1.47603, + "38": 1.22908, + "39": 2.05448, + "40": 1.56999, + "41": 1.69715, + "42": 2.26001, + "43": 1.70406, + "44": 1.71274, + "45": 2.04011, + "46": 1.45177, + "47": 1.64853, + "48": 1.8101, + "49": 1.35245, + "50": 1.73124 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json index 19a3e19f69c..9fe04ae9661 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94183, - "2": 10.93668, - "3": 10.93422, - "4": 10.94286, - "5": 10.93217, - "6": 10.93327, - "7": 10.93827, - "8": 10.93702, - "9": 10.93302, - "10": 10.9359, - "11": 10.92315, - "12": 10.92245, - "13": 10.92251, - "14": 10.91535, - "15": 10.89063, - "16": 10.88921, - "17": 10.89984, - "18": 10.87856, - "19": 10.87324, - "20": 10.79044, - "21": 10.78086, - "22": 10.77833, - "23": 10.77616, - "24": 10.73356, - "25": 10.74367, - "26": 10.72368, - "27": 10.69196, - "28": 10.62234, - "29": 10.60126, - "30": 10.57969, - "31": 10.56879, - "32": 10.54771, - "33": 10.51956, - "34": 10.48216, - "35": 10.48743, - "36": 10.46841, - "37": 10.43367, - "38": 10.43558, + "1": 10.94173, + "2": 10.9367, + "3": 10.93428, + "4": 10.94283, + "5": 10.93207, + "6": 10.93333, + "7": 10.93831, + "8": 10.93696, + "9": 10.93307, + "10": 10.93586, + "11": 10.92319, + "12": 10.9224, + "13": 10.92256, + "14": 10.91538, + "15": 10.89071, + "16": 10.88927, + "17": 10.89987, + "18": 10.87853, + "19": 10.87329, + "20": 10.79041, + "21": 10.78094, + "22": 10.77835, + "23": 10.7762, + "24": 10.7336, + "25": 10.74369, + "26": 10.7237, + "27": 10.69206, + "28": 10.62233, + "29": 10.60131, + "30": 10.57977, + "31": 10.56888, + "32": 10.5478, + "33": 10.51962, + "34": 10.48215, + "35": 10.48741, + "36": 10.46839, + "37": 10.43365, + "38": 10.43564, "39": 10.39937, - "40": 10.3877, - "41": 10.36771, - "42": 10.34241, - "43": 10.31723, - "44": 10.28162, - "45": 10.29681, - "46": 10.25729, - "47": 10.24027, - "48": 10.19352, - "49": 10.19318, - "50": 10.20171, - "51": 10.19541, - "52": 10.14635, - "53": 10.15808, - "54": 10.12048, - "55": 10.0895, - "56": 10.11775, - "57": 10.10041, - "58": 10.11737, - "59": 10.05933, - "60": 10.07204, - "61": 10.02672, - "62": 9.99427, - "63": 10.06736, - "64": 10.02126, - "65": 9.99228, - "66": 10.02171, - "67": 9.9873, - "68": 9.9525, - "69": 9.97128, - "70": 9.96017, - "71": 9.97797, - "72": 9.94778, - "73": 9.94274, - "74": 9.93232, - "75": 9.8983, - "76": 9.93909, - "77": 9.92997, - "78": 9.87615, - "79": 9.88443, - "80": 9.89726, - "81": 9.91791, - "82": 9.86013, - "83": 9.81875, - "84": 9.75664, - "85": 9.74768, - "86": 9.84735, - "87": 9.87997, - "88": 9.85067, - "89": 9.78575, - "90": 9.77121, - "91": 9.78203, - "92": 9.77021, - "93": 9.70718, - "94": 9.77824, - "95": 9.77246, - "96": 9.75758, - "97": 9.69483, - "98": 9.72153, - "99": 9.76727, - "100": 9.65545 + "40": 10.38779, + "41": 10.3678, + "42": 10.34247, + "43": 10.3173, + "44": 10.2817, + "45": 10.29693, + "46": 10.2573, + "47": 10.24033, + "48": 10.19361, + "49": 10.19314, + "50": 10.20179, + "51": 10.19543, + "52": 10.14638, + "53": 10.15816, + "54": 10.12057, + "55": 10.08956, + "56": 10.11779, + "57": 10.10045, + "58": 10.11735, + "59": 10.05935, + "60": 10.07209, + "61": 10.02671, + "62": 9.99432, + "63": 10.06742, + "64": 10.02132, + "65": 9.99232, + "66": 10.02172, + "67": 9.98728, + "68": 9.95255, + "69": 9.97132, + "70": 9.96021, + "71": 9.978, + "72": 9.94782, + "73": 9.9428, + "74": 9.93234, + "75": 9.89828, + "76": 9.93916, + "77": 9.93003, + "78": 9.87622, + "79": 9.88448, + "80": 9.89728, + "81": 9.91798, + "82": 9.86017, + "83": 9.8188, + "84": 9.75672, + "85": 9.74773, + "86": 9.8474, + "87": 9.87995, + "88": 9.8507, + "89": 9.78573, + "90": 9.7713, + "91": 9.78213, + "92": 9.77027, + "93": 9.70721, + "94": 9.77827, + "95": 9.77248, + "96": 9.7576, + "97": 9.69485, + "98": 9.72159, + "99": 9.76733, + "100": 9.65541 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1932.0, - "2": 1782.0, - "3": 1803.0, - "4": 1779.0, - "5": 1706.0, - "6": 1702.0, - "7": 2051.0, - "8": 1849.0, - "9": 1770.0, - "10": 1819.0, - "11": 1731.0, - "12": 1719.0, - "13": 1856.0, - "14": 1782.0, - "15": 1692.0, - "16": 1689.0, - "17": 1773.0, - "18": 1767.0, - "19": 1690.0, - "20": 1685.0, - "21": 1806.0, - "22": 1759.0, - "23": 1726.0, - "24": 1915.0, - "25": 1782.0, - "26": 1844.0, - "27": 1846.0, - "28": 1890.0, - "29": 1998.0, - "30": 1817.0, - "31": 1964.0, - "32": 1930.0, - "33": 1890.0, - "34": 2110.0, - "35": 1983.0, - "36": 1942.0, - "37": 2194.0, - "38": 2192.0, - "39": 2211.0, - "40": 2297.0, - "41": 2365.0, - "42": 2015.0, - "43": 2251.0, - "44": 2176.0, - "45": 2512.0, - "46": 2444.0, - "47": 2370.0, - "48": 2550.0, - "49": 2703.0, - "50": 2458.0, - "51": 2397.0, - "52": 2629.0, - "53": 2525.0, - "54": 2717.0, - "55": 2504.0, - "56": 2645.0, - "57": 2250.0, - "58": 3458.0, - "59": 2793.0, - "60": 2877.0, - "61": 2840.0, - "62": 3147.0, - "63": 3355.0, - "64": 3479.0, - "65": 2608.0, - "66": 3104.0, - "67": 3866.0, - "68": 3432.0, - "69": 2824.0, - "70": 3095.0, - "71": 3155.0, - "72": 3035.0, - "73": 3269.0, - "74": 3078.0, - "75": 3120.0, - "76": 3208.0, - "77": 3687.0, - "78": 3199.0, - "79": 3266.0, - "80": 3055.0, - "81": 3371.0, - "82": 3092.0, - "83": 3167.0, - "84": 3024.0, - "85": 2726.0, - "86": 3084.0, - "87": 2855.0, - "88": 2969.0, - "89": 2879.0, - "90": 3509.0, - "91": 3079.0, - "92": 2895.0, - "93": 3146.0, - "94": 3270.0, - "95": 3457.0, - "96": 3414.0, - "97": 3627.0, - "98": 3677.0, - "99": 3395.0, - "100": 3317.0 + "1": 1834.0, + "2": 1767.0, + "3": 1730.0, + "4": 1718.0, + "5": 1745.0, + "6": 1812.0, + "7": 2019.0, + "8": 1661.0, + "9": 1808.0, + "10": 1742.0, + "11": 1759.0, + "12": 1717.0, + "13": 1828.0, + "14": 1851.0, + "15": 1670.0, + "16": 1752.0, + "17": 1866.0, + "18": 1922.0, + "19": 1800.0, + "20": 1695.0, + "21": 1835.0, + "22": 1744.0, + "23": 1781.0, + "24": 1957.0, + "25": 1764.0, + "26": 1808.0, + "27": 1767.0, + "28": 1877.0, + "29": 1996.0, + "30": 1928.0, + "31": 1903.0, + "32": 2002.0, + "33": 1968.0, + "34": 2043.0, + "35": 2087.0, + "36": 2009.0, + "37": 2199.0, + "38": 2209.0, + "39": 2078.0, + "40": 2323.0, + "41": 2336.0, + "42": 1940.0, + "43": 2309.0, + "44": 2132.0, + "45": 2546.0, + "46": 2418.0, + "47": 2473.0, + "48": 2585.0, + "49": 2747.0, + "50": 2502.0, + "51": 2393.0, + "52": 2743.0, + "53": 2568.0, + "54": 2781.0, + "55": 2481.0, + "56": 2665.0, + "57": 2103.0, + "58": 3500.0, + "59": 2777.0, + "60": 2934.0, + "61": 2632.0, + "62": 3110.0, + "63": 3313.0, + "64": 3470.0, + "65": 2637.0, + "66": 3039.0, + "67": 3875.0, + "68": 3479.0, + "69": 2856.0, + "70": 3158.0, + "71": 3152.0, + "72": 2833.0, + "73": 3290.0, + "74": 3103.0, + "75": 3153.0, + "76": 3113.0, + "77": 3715.0, + "78": 3120.0, + "79": 3272.0, + "80": 3059.0, + "81": 3319.0, + "82": 3099.0, + "83": 3220.0, + "84": 2980.0, + "85": 2688.0, + "86": 3030.0, + "87": 2823.0, + "88": 3083.0, + "89": 2776.0, + "90": 3502.0, + "91": 2971.0, + "92": 2911.0, + "93": 3103.0, + "94": 3104.0, + "95": 3332.0, + "96": 3474.0, + "97": 3622.0, + "98": 3615.0, + "99": 3273.0, + "100": 3274.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.24191, - "3": 0.13299, - "4": 0.11456, - "5": 0.11433, - "6": 0.11378, - "7": 0.11251, - "8": 0.11388, - "9": 0.11346, - "10": 0.11431, - "11": 0.11494, - "12": 0.11523, - "13": 0.1146, - "14": 0.11574, - "15": 0.11467, - "16": 0.11397, - "17": 0.11476, - "18": 0.11526, - "19": 0.11621, - "20": 0.11457, - "21": 0.11612, - "22": 0.11355, - "23": 0.11461, - "24": 0.11534, - "25": 0.11606, - "26": 0.11536, - "27": 0.11483, - "28": 0.11529, - "29": 0.11485, - "30": 0.11585, - "31": 0.11506, - "32": 0.11534, - "33": 0.11506, - "34": 0.11583, - "35": 0.1132, - "36": 0.11311, - "37": 0.11305, - "38": 0.11273, - "39": 0.11331, - "40": 0.11254, - "41": 0.11274, - "42": 0.11302, - "43": 0.11354, - "44": 0.11329, - "45": 0.11394, - "46": 0.11289, - "47": 0.11321, - "48": 0.11301, - "49": 0.11353, - "50": 0.11308, - "51": 0.46962, - "52": 0.11612, - "53": 0.1204, - "54": 0.11492, - "55": 0.11499, - "56": 0.11501, - "57": 0.11437, - "58": 0.11402, - "59": 0.11286, - "60": 0.11331, - "61": 0.11546, - "62": 0.11335, - "63": 0.11477, - "64": 0.11393, - "65": 0.11467, - "66": 0.11454, - "67": 0.11445, - "68": 0.11343, - "69": 0.11709, - "70": 0.11601, - "71": 0.11527, - "72": 0.11441, - "73": 0.11443, - "74": 0.1158, - "75": 0.11366, - "76": 0.11464, - "77": 0.11371, - "78": 0.11502, - "79": 0.11393, - "80": 0.11406, - "81": 0.1149, - "82": 0.11479, - "83": 0.11444, - "84": 0.11438, - "85": 0.11446, - "86": 0.11399, - "87": 0.11447, - "88": 0.11495, - "89": 0.11564, - "90": 0.11438, - "91": 0.11515, - "92": 0.11432, - "93": 0.11443, - "94": 0.1161, - "95": 0.11616, - "96": 0.11535, - "97": 0.11471, - "98": 0.11624, - "99": 0.1169, - "100": 0.11528 + "2": 10.6197, + "3": 0.15624, + "4": 0.10432, + "5": 0.09907, + "6": 0.10036, + "7": 0.10006, + "8": 0.09745, + "9": 0.09944, + "10": 0.26153, + "11": 0.09956, + "12": 0.10548, + "13": 0.09701, + "14": 0.09894, + "15": 0.09963, + "16": 0.1016, + "17": 0.104, + "18": 0.10126, + "19": 0.2664, + "20": 0.10456, + "21": 0.10092, + "22": 0.10244, + "23": 0.10279, + "24": 0.1006, + "25": 0.10156, + "26": 0.10082, + "27": 0.10064, + "28": 0.09771, + "29": 0.10287, + "30": 0.10431, + "31": 0.10035, + "32": 0.10195, + "33": 0.10187, + "34": 0.10073, + "35": 0.10122, + "36": 0.10559, + "37": 0.10428, + "38": 0.10258, + "39": 0.24502, + "40": 0.10222, + "41": 0.10167, + "42": 0.10056, + "43": 0.10252, + "44": 0.10308, + "45": 0.10163, + "46": 0.09943, + "47": 0.14689, + "48": 0.20501, + "49": 0.10224, + "50": 0.10043, + "51": 0.52242, + "52": 0.10605, + "53": 0.10117, + "54": 0.21153, + "55": 0.10081, + "56": 0.10273, + "57": 0.10427, + "58": 0.1006, + "59": 0.10261, + "60": 0.10061, + "61": 0.10045, + "62": 0.10223, + "63": 0.10258, + "64": 0.10095, + "65": 0.1005, + "66": 0.10069, + "67": 0.10077, + "68": 0.1021, + "69": 0.10009, + "70": 0.10098, + "71": 0.10091, + "72": 0.10246, + "73": 0.10241, + "74": 0.09998, + "75": 0.10125, + "76": 0.10036, + "77": 0.10155, + "78": 0.0999, + "79": 0.09908, + "80": 0.10189, + "81": 0.1018, + "82": 0.09986, + "83": 0.10129, + "84": 0.10318, + "85": 0.10257, + "86": 0.10453, + "87": 0.10284, + "88": 0.10425, + "89": 0.16106, + "90": 0.10083, + "91": 0.10296, + "92": 0.09813, + "93": 0.10325, + "94": 0.10042, + "95": 0.10298, + "96": 0.10158, + "97": 0.10307, + "98": 0.10061, + "99": 0.10098, + "100": 0.10142 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json index f6aa3b73aee..30af5ffd86c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95856, + "1": 10.96462, + "2": 10.95232, + "3": 10.95858, "4": 10.95193, - "5": 10.95264, - "6": 10.95378, + "5": 10.95261, + "6": 10.95383, "7": 10.95159, - "8": 10.95135, - "9": 10.95869, - "10": 10.94454, - "11": 10.93683, - "12": 10.94341, - "13": 10.94309, - "14": 10.94286, - "15": 10.91495, - "16": 10.90027, - "17": 10.91108, - "18": 10.90589, - "19": 10.90126, - "20": 10.80771, - "21": 10.79443, - "22": 10.80462, - "23": 10.78963, - "24": 10.77416, - "25": 10.7606, - "26": 10.75391, - "27": 10.7187, - "28": 10.63859, - "29": 10.60793, - "30": 10.58595, - "31": 10.59034, - "32": 10.56919, - "33": 10.53723, + "8": 10.95138, + "9": 10.95865, + "10": 10.94448, + "11": 10.93684, + "12": 10.94335, + "13": 10.94307, + "14": 10.94284, + "15": 10.91488, + "16": 10.90025, + "17": 10.911, + "18": 10.90585, + "19": 10.90125, + "20": 10.80769, + "21": 10.7944, + "22": 10.80464, + "23": 10.78956, + "24": 10.77419, + "25": 10.76059, + "26": 10.75389, + "27": 10.71867, + "28": 10.63858, + "29": 10.60799, + "30": 10.58598, + "31": 10.59032, + "32": 10.56926, + "33": 10.5373, "34": 10.49524, - "35": 10.49635, - "36": 10.48327, - "37": 10.44677, - "38": 10.4482, + "35": 10.49637, + "36": 10.48329, + "37": 10.44678, + "38": 10.44825, "39": 10.41533, - "40": 10.39307, - "41": 10.37226, - "42": 10.35597, + "40": 10.3931, + "41": 10.37237, + "42": 10.35606, "43": 10.32385, - "44": 10.31083, - "45": 10.30676, - "46": 10.266, - "47": 10.2582, - "48": 10.20966, - "49": 10.20575, - "50": 10.21353, - "51": 10.21324, - "52": 10.15841, - "53": 10.16271, - "54": 10.12809, + "44": 10.31086, + "45": 10.30682, + "46": 10.26607, + "47": 10.25827, + "48": 10.20968, + "49": 10.20578, + "50": 10.21359, + "51": 10.21329, + "52": 10.15843, + "53": 10.16273, + "54": 10.12814, "55": 10.09863, - "56": 10.12108, - "57": 10.10886, - "58": 10.11352, - "59": 10.05657, - "60": 10.07841, - "61": 10.03375, - "62": 10.00109, - "63": 10.06759, - "64": 10.01686, + "56": 10.12107, + "57": 10.10884, + "58": 10.11353, + "59": 10.0566, + "60": 10.07848, + "61": 10.03382, + "62": 10.00115, + "63": 10.06764, + "64": 10.0169, "65": 9.98864, - "66": 10.02938, - "67": 9.99989, - "68": 9.96572, - "69": 9.97412, - "70": 9.95973, - "71": 9.98131, - "72": 9.96658, - "73": 9.94918, - "74": 9.94054, - "75": 9.91022, - "76": 9.93992, - "77": 9.93684, + "66": 10.02946, + "67": 9.99988, + "68": 9.96578, + "69": 9.97419, + "70": 9.95978, + "71": 9.98134, + "72": 9.96659, + "73": 9.94919, + "74": 9.94059, + "75": 9.91026, + "76": 9.93997, + "77": 9.93687, "78": 9.88872, - "79": 9.8896, - "80": 9.90383, - "81": 9.92953, - "82": 9.86789, - "83": 9.83326, - "84": 9.77449, - "85": 9.75504, - "86": 9.8559, - "87": 9.87809, - "88": 9.84953, - "89": 9.78708, - "90": 9.78521, - "91": 9.78362, - "92": 9.77827, - "93": 9.70912, - "94": 9.78142, - "95": 9.76786, - "96": 9.75262, - "97": 9.7003, - "98": 9.71823, + "79": 9.88964, + "80": 9.90392, + "81": 9.92963, + "82": 9.86795, + "83": 9.8333, + "84": 9.77456, + "85": 9.75509, + "86": 9.85593, + "87": 9.87807, + "88": 9.84954, + "89": 9.78714, + "90": 9.78526, + "91": 9.78366, + "92": 9.77828, + "93": 9.70916, + "94": 9.78145, + "95": 9.76788, + "96": 9.75266, + "97": 9.70036, + "98": 9.7183, "99": 9.77348, - "100": 9.6534 + "100": 9.65348 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1840.0, - "4": 1776.0, - "5": 1738.0, - "6": 1754.0, - "7": 1820.0, - "8": 1756.0, - "9": 1687.0, - "10": 1731.0, - "11": 1692.0, - "12": 1629.0, - "13": 1860.0, - "14": 1863.0, - "15": 1580.0, - "16": 1765.0, - "17": 1790.0, - "18": 1790.0, - "19": 1722.0, - "20": 1727.0, - "21": 1833.0, - "22": 1757.0, - "23": 1805.0, - "24": 1881.0, - "25": 1782.0, - "26": 1795.0, - "27": 1810.0, - "28": 1781.0, - "29": 1842.0, - "30": 1817.0, - "31": 1973.0, - "32": 1968.0, - "33": 1939.0, - "34": 1961.0, - "35": 2034.0, - "36": 2031.0, - "37": 2203.0, - "38": 2042.0, - "39": 2123.0, - "40": 2164.0, - "41": 2272.0, - "42": 1959.0, - "43": 2318.0, - "44": 2152.0, - "45": 2439.0, - "46": 2332.0, - "47": 2369.0, - "48": 2574.0, - "49": 2872.0, - "50": 2550.0, - "51": 2403.0, - "52": 2649.0, - "53": 2555.0, - "54": 2693.0, - "55": 2363.0, - "56": 2705.0, - "57": 2210.0, - "58": 3467.0, - "59": 2889.0, - "60": 2925.0, - "61": 2651.0, - "62": 3162.0, - "63": 3222.0, - "64": 3604.0, - "65": 2772.0, - "66": 2971.0, - "67": 3800.0, - "68": 3304.0, - "69": 2983.0, - "70": 3327.0, - "71": 3168.0, - "72": 2835.0, - "73": 3418.0, - "74": 3295.0, - "75": 3027.0, - "76": 3280.0, - "77": 3707.0, - "78": 3342.0, - "79": 3320.0, - "80": 3060.0, - "81": 3370.0, - "82": 3036.0, - "83": 3024.0, - "84": 2865.0, - "85": 2793.0, - "86": 3030.0, - "87": 2852.0, - "88": 3000.0, - "89": 3197.0, - "90": 3708.0, - "91": 2908.0, - "92": 3069.0, - "93": 3103.0, - "94": 2906.0, - "95": 3462.0, - "96": 3323.0, - "97": 3535.0, - "98": 3390.0, - "99": 3077.0, - "100": 3122.0 + "1": 1759.0, + "2": 1644.0, + "3": 1707.0, + "4": 1731.0, + "5": 1784.0, + "6": 1746.0, + "7": 1886.0, + "8": 1676.0, + "9": 1742.0, + "10": 1790.0, + "11": 1710.0, + "12": 1691.0, + "13": 1829.0, + "14": 1930.0, + "15": 1544.0, + "16": 1792.0, + "17": 1770.0, + "18": 1843.0, + "19": 1713.0, + "20": 1685.0, + "21": 1765.0, + "22": 1728.0, + "23": 1696.0, + "24": 1902.0, + "25": 1684.0, + "26": 1774.0, + "27": 1732.0, + "28": 1812.0, + "29": 1968.0, + "30": 1819.0, + "31": 1993.0, + "32": 1940.0, + "33": 1927.0, + "34": 1973.0, + "35": 2130.0, + "36": 1984.0, + "37": 2200.0, + "38": 2051.0, + "39": 2217.0, + "40": 2276.0, + "41": 2308.0, + "42": 2065.0, + "43": 2317.0, + "44": 2204.0, + "45": 2395.0, + "46": 2409.0, + "47": 2404.0, + "48": 2601.0, + "49": 2895.0, + "50": 2543.0, + "51": 2440.0, + "52": 2670.0, + "53": 2604.0, + "54": 2598.0, + "55": 2549.0, + "56": 2611.0, + "57": 2194.0, + "58": 3601.0, + "59": 2800.0, + "60": 2976.0, + "61": 2615.0, + "62": 3120.0, + "63": 3048.0, + "64": 3516.0, + "65": 2729.0, + "66": 3001.0, + "67": 3661.0, + "68": 3390.0, + "69": 2960.0, + "70": 3335.0, + "71": 3221.0, + "72": 3007.0, + "73": 3363.0, + "74": 3229.0, + "75": 3100.0, + "76": 3237.0, + "77": 3528.0, + "78": 3363.0, + "79": 3337.0, + "80": 2968.0, + "81": 3560.0, + "82": 3006.0, + "83": 3132.0, + "84": 2916.0, + "85": 2723.0, + "86": 3066.0, + "87": 2765.0, + "88": 3111.0, + "89": 3277.0, + "90": 3660.0, + "91": 2861.0, + "92": 3046.0, + "93": 3143.0, + "94": 2927.0, + "95": 3412.0, + "96": 3335.0, + "97": 3499.0, + "98": 3440.0, + "99": 2979.0, + "100": 3130.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.76577, - "3": 0.10888, - "4": 0.10801, - "5": 0.10507, - "6": 0.10756, - "7": 0.10683, - "8": 0.11011, - "9": 0.10535, - "10": 0.10554, - "11": 0.10849, - "12": 0.107, - "13": 0.10477, - "14": 0.10681, - "15": 0.10597, - "16": 0.10862, - "17": 0.10543, - "18": 0.11036, - "19": 0.10405, - "20": 0.1044, - "21": 0.10336, - "22": 0.10377, - "23": 0.1077, - "24": 0.10989, - "25": 0.10804, - "26": 0.10778, - "27": 0.11124, - "28": 0.10764, - "29": 0.11152, - "30": 0.10649, - "31": 0.11059, - "32": 0.10649, - "33": 0.10428, - "34": 0.1065, - "35": 0.10718, - "36": 0.11006, - "37": 0.11092, - "38": 0.10975, - "39": 0.10918, - "40": 0.10574, - "41": 0.108, - "42": 0.10816, - "43": 0.10971, - "44": 0.10792, - "45": 0.10927, - "46": 0.10431, - "47": 0.10912, - "48": 0.10941, - "49": 0.10696, - "50": 0.10682, - "51": 0.16269, - "52": 0.10898, - "53": 0.10613, - "54": 0.10907, - "55": 0.1061, - "56": 0.10726, - "57": 0.10671, - "58": 0.10813, - "59": 0.11101, - "60": 0.10731, - "61": 0.11075, - "62": 0.11072, - "63": 0.10816, - "64": 0.10653, - "65": 0.10565, - "66": 0.1087, - "67": 0.10738, - "68": 0.11142, - "69": 0.10798, - "70": 0.1069, - "71": 0.10471, - "72": 0.10769, - "73": 0.1079, - "74": 0.10753, - "75": 0.10839, - "76": 0.10612, - "77": 0.10624, - "78": 0.10666, - "79": 0.10679, - "80": 0.10858, - "81": 0.10683, - "82": 0.10569, - "83": 0.1092, - "84": 0.1069, - "85": 0.10972, - "86": 0.10149, - "87": 0.10613, - "88": 0.1073, - "89": 0.10598, - "90": 0.10894, - "91": 0.10465, - "92": 0.10771, - "93": 0.10564, - "94": 0.10242, - "95": 0.1048, - "96": 0.1089, - "97": 0.10597, - "98": 0.10765, - "99": 0.1079, - "100": 0.10669 + "2": 9.54844, + "3": 0.1781, + "4": 0.11295, + "5": 0.11294, + "6": 0.11341, + "7": 0.11324, + "8": 0.1128, + "9": 0.11576, + "10": 0.11365, + "11": 0.11478, + "12": 0.11391, + "13": 0.11015, + "14": 0.16668, + "15": 0.11292, + "16": 0.11776, + "17": 0.11326, + "18": 0.11207, + "19": 0.11153, + "20": 0.11236, + "21": 0.11251, + "22": 0.11234, + "23": 0.11929, + "24": 0.11623, + "25": 0.11273, + "26": 0.11289, + "27": 0.11233, + "28": 0.11243, + "29": 0.11426, + "30": 0.11567, + "31": 0.11674, + "32": 0.11503, + "33": 0.11778, + "34": 0.11032, + "35": 0.11119, + "36": 0.11527, + "37": 0.11265, + "38": 0.11371, + "39": 0.11568, + "40": 0.11265, + "41": 0.11396, + "42": 0.11643, + "43": 0.11316, + "44": 0.11514, + "45": 0.11458, + "46": 0.11382, + "47": 0.11628, + "48": 0.11237, + "49": 0.11373, + "50": 0.11387, + "51": 0.16539, + "52": 0.11456, + "53": 0.11432, + "54": 0.11601, + "55": 0.1118, + "56": 0.11631, + "57": 0.11415, + "58": 0.11344, + "59": 0.11563, + "60": 0.11157, + "61": 0.11317, + "62": 0.11602, + "63": 0.11385, + "64": 0.11819, + "65": 0.11189, + "66": 0.11544, + "67": 0.1134, + "68": 0.11957, + "69": 0.11351, + "70": 0.11321, + "71": 0.11278, + "72": 0.11358, + "73": 0.11656, + "74": 0.11295, + "75": 0.11508, + "76": 0.11355, + "77": 0.11427, + "78": 0.11547, + "79": 0.11329, + "80": 0.1147, + "81": 0.11145, + "82": 0.11684, + "83": 0.11531, + "84": 0.11305, + "85": 0.11309, + "86": 0.10914, + "87": 0.11223, + "88": 0.11507, + "89": 0.11343, + "90": 0.11272, + "91": 0.11523, + "92": 0.11629, + "93": 0.11425, + "94": 0.11136, + "95": 0.11405, + "96": 0.11572, + "97": 0.11169, + "98": 0.11113, + "99": 0.11319, + "100": 0.11453 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json index cb3e6b857a1..d93e4a0829c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.94232, - "5": 10.93349, - "6": 10.93269, - "7": 10.93796, - "8": 10.93356, - "9": 10.93555, - "10": 10.93692, - "11": 10.92585, - "12": 10.92412, - "13": 10.92504, - "14": 10.91248, - "15": 10.89335, - "16": 10.88815, - "17": 10.89968, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.9423, + "5": 10.93351, + "6": 10.93267, + "7": 10.93789, + "8": 10.93352, + "9": 10.93554, + "10": 10.93686, + "11": 10.92579, + "12": 10.9241, + "13": 10.92503, + "14": 10.91244, + "15": 10.89334, + "16": 10.88825, + "17": 10.89966, "18": 10.87789, - "19": 10.87549, - "20": 10.79108, - "21": 10.78001, - "22": 10.77459, - "23": 10.77485, - "24": 10.73559, - "25": 10.74282, - "26": 10.72347, - "27": 10.6955, - "28": 10.62303, - "29": 10.60325, - "30": 10.57991, - "31": 10.56891, - "32": 10.54647, - "33": 10.51723, - "34": 10.48161, - "35": 10.48816, - "36": 10.46791, - "37": 10.43373, - "38": 10.43848, - "39": 10.40028, - "40": 10.3903, - "41": 10.36588, - "42": 10.34114, - "43": 10.3191, - "44": 10.28097, - "45": 10.2984, - "46": 10.25723, - "47": 10.24193, - "48": 10.19501, - "49": 10.19305, - "50": 10.20072, - "51": 10.19623, - "52": 10.14789, - "53": 10.15718, - "54": 10.12042, - "55": 10.09009, + "19": 10.87552, + "20": 10.79109, + "21": 10.78002, + "22": 10.77465, + "23": 10.77492, + "24": 10.73562, + "25": 10.7428, + "26": 10.72344, + "27": 10.69551, + "28": 10.62308, + "29": 10.60327, + "30": 10.57993, + "31": 10.56896, + "32": 10.54649, + "33": 10.51737, + "34": 10.48165, + "35": 10.48815, + "36": 10.468, + "37": 10.43369, + "38": 10.4385, + "39": 10.40031, + "40": 10.39035, + "41": 10.36597, + "42": 10.34125, + "43": 10.31908, + "44": 10.281, + "45": 10.29848, + "46": 10.25735, + "47": 10.24198, + "48": 10.19505, + "49": 10.19309, + "50": 10.20075, + "51": 10.19627, + "52": 10.14797, + "53": 10.1572, + "54": 10.12045, + "55": 10.09012, "56": 10.1185, - "57": 10.10154, - "58": 10.11659, - "59": 10.06156, - "60": 10.07213, - "61": 10.02683, - "62": 9.9928, - "63": 10.06735, + "57": 10.10153, + "58": 10.11664, + "59": 10.06158, + "60": 10.07214, + "61": 10.02685, + "62": 9.99289, + "63": 10.0674, "64": 10.02093, - "65": 9.99324, - "66": 10.0227, - "67": 9.98726, - "68": 9.95238, - "69": 9.97227, - "70": 9.95945, - "71": 9.97771, - "72": 9.94822, - "73": 9.94072, - "74": 9.9307, - "75": 9.89817, - "76": 9.9397, - "77": 9.9293, - "78": 9.87561, - "79": 9.88327, - "80": 9.89532, - "81": 9.91711, - "82": 9.85934, - "83": 9.81914, - "84": 9.75546, - "85": 9.74598, - "86": 9.847, - "87": 9.87922, - "88": 9.85037, - "89": 9.78552, - "90": 9.77222, - "91": 9.78185, - "92": 9.76913, - "93": 9.70722, - "94": 9.77765, - "95": 9.77216, - "96": 9.75877, - "97": 9.69519, - "98": 9.72098, - "99": 9.76711, - "100": 9.65463 + "65": 9.99329, + "66": 10.02279, + "67": 9.98731, + "68": 9.95243, + "69": 9.97239, + "70": 9.95946, + "71": 9.97778, + "72": 9.9483, + "73": 9.9408, + "74": 9.93077, + "75": 9.89819, + "76": 9.93977, + "77": 9.92936, + "78": 9.87568, + "79": 9.8833, + "80": 9.89538, + "81": 9.91715, + "82": 9.85935, + "83": 9.81917, + "84": 9.75554, + "85": 9.74603, + "86": 9.84705, + "87": 9.87929, + "88": 9.85046, + "89": 9.78555, + "90": 9.7723, + "91": 9.78186, + "92": 9.76916, + "93": 9.70728, + "94": 9.77774, + "95": 9.77221, + "96": 9.75888, + "97": 9.69527, + "98": 9.72103, + "99": 9.76715, + "100": 9.65471 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1823.0, - "4": 1751.0, - "5": 1765.0, - "6": 1747.0, + "1": 1791.0, + "2": 1794.0, + "3": 1792.0, + "4": 1665.0, + "5": 1760.0, + "6": 1738.0, "7": 2058.0, - "8": 1609.0, - "9": 1791.0, - "10": 1746.0, - "11": 1671.0, - "12": 1691.0, - "13": 1884.0, - "14": 1910.0, - "15": 1701.0, - "16": 1714.0, - "17": 1896.0, - "18": 1865.0, - "19": 1790.0, - "20": 1786.0, - "21": 1826.0, - "22": 1753.0, - "23": 1655.0, - "24": 1889.0, - "25": 1720.0, - "26": 1854.0, - "27": 1720.0, - "28": 1872.0, - "29": 1931.0, - "30": 1873.0, - "31": 2004.0, - "32": 1975.0, - "33": 1992.0, - "34": 2001.0, - "35": 2033.0, - "36": 1907.0, - "37": 2178.0, - "38": 2172.0, - "39": 2174.0, - "40": 2324.0, - "41": 2413.0, - "42": 1986.0, - "43": 2354.0, - "44": 2233.0, - "45": 2482.0, - "46": 2411.0, - "47": 2288.0, + "8": 1660.0, + "9": 1819.0, + "10": 1668.0, + "11": 1678.0, + "12": 1671.0, + "13": 1904.0, + "14": 1855.0, + "15": 1633.0, + "16": 1676.0, + "17": 1850.0, + "18": 1852.0, + "19": 1680.0, + "20": 1701.0, + "21": 1721.0, + "22": 1703.0, + "23": 1684.0, + "24": 1886.0, + "25": 1803.0, + "26": 1906.0, + "27": 1789.0, + "28": 1775.0, + "29": 1957.0, + "30": 1897.0, + "31": 1968.0, + "32": 1987.0, + "33": 2015.0, + "34": 1994.0, + "35": 2010.0, + "36": 1915.0, + "37": 2261.0, + "38": 2056.0, + "39": 2112.0, + "40": 2336.0, + "41": 2314.0, + "42": 2051.0, + "43": 2405.0, + "44": 2206.0, + "45": 2437.0, + "46": 2410.0, + "47": 2351.0, "48": 2670.0, - "49": 2785.0, - "50": 2757.0, - "51": 2528.0, - "52": 2712.0, - "53": 2618.0, - "54": 2827.0, - "55": 2526.0, - "56": 2672.0, - "57": 2172.0, - "58": 3466.0, - "59": 2770.0, - "60": 2893.0, - "61": 2653.0, - "62": 3098.0, - "63": 3284.0, - "64": 3400.0, - "65": 2751.0, - "66": 3072.0, - "67": 3801.0, - "68": 3389.0, - "69": 2897.0, - "70": 3096.0, - "71": 3056.0, - "72": 2962.0, - "73": 3260.0, - "74": 3118.0, - "75": 3090.0, - "76": 3252.0, - "77": 3597.0, - "78": 3083.0, - "79": 3233.0, - "80": 2958.0, - "81": 3295.0, - "82": 3257.0, - "83": 3143.0, - "84": 3047.0, - "85": 2752.0, - "86": 3251.0, - "87": 2866.0, - "88": 3149.0, - "89": 2803.0, - "90": 3609.0, - "91": 3035.0, - "92": 2997.0, - "93": 3106.0, - "94": 3118.0, - "95": 3399.0, - "96": 3534.0, - "97": 3611.0, - "98": 3729.0, - "99": 3121.0, - "100": 3262.0 + "49": 2792.0, + "50": 2550.0, + "51": 2466.0, + "52": 2738.0, + "53": 2570.0, + "54": 2791.0, + "55": 2465.0, + "56": 2782.0, + "57": 2212.0, + "58": 3482.0, + "59": 2828.0, + "60": 2865.0, + "61": 2695.0, + "62": 3143.0, + "63": 3205.0, + "64": 3346.0, + "65": 2665.0, + "66": 3013.0, + "67": 3770.0, + "68": 3440.0, + "69": 2919.0, + "70": 3019.0, + "71": 3032.0, + "72": 2969.0, + "73": 3343.0, + "74": 3143.0, + "75": 3120.0, + "76": 3156.0, + "77": 3601.0, + "78": 3094.0, + "79": 3297.0, + "80": 3066.0, + "81": 3228.0, + "82": 3232.0, + "83": 3248.0, + "84": 3002.0, + "85": 2885.0, + "86": 3133.0, + "87": 2915.0, + "88": 3126.0, + "89": 2864.0, + "90": 3532.0, + "91": 2981.0, + "92": 2952.0, + "93": 3250.0, + "94": 3233.0, + "95": 3500.0, + "96": 3551.0, + "97": 3640.0, + "98": 3738.0, + "99": 3123.0, + "100": 3338.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.86698, - "3": 2.23709, - "4": 1.93789, - "5": 1.54945, - "6": 1.8486, - "7": 1.56946, - "8": 1.90325, - "9": 2.25159, - "10": 1.61854, - "11": 1.86873, - "12": 1.51085, - "13": 1.65829, - "14": 1.50138, - "15": 1.37256, - "16": 1.6651, - "17": 2.38708, - "18": 1.47357, - "19": 1.73005, - "20": 1.71666, - "21": 1.85755, - "22": 1.201, - "23": 1.92128, - "24": 1.29664, - "25": 2.19689, - "26": 2.1285, - "27": 1.55775, - "28": 1.70244, - "29": 1.58319, - "30": 1.54706, - "31": 1.26425, - "32": 1.94276, - "33": 1.38332, - "34": 1.46561, - "35": 1.98009, - "36": 1.949, - "37": 1.6834, - "38": 1.32733, - "39": 2.09247, - "40": 1.55836, - "41": 1.76807, - "42": 2.34285, - "43": 1.58718, - "44": 1.63324, - "45": 1.93623, - "46": 1.62651, - "47": 1.86707, - "48": 1.87844, - "49": 1.24183, - "50": 1.7234, - "51": 1.58703, - "52": 1.75205, - "53": 1.63938, - "54": 1.82167, - "55": 1.76283, - "56": 2.35548, - "57": 1.26384, - "58": 2.15886, - "59": 1.68252, - "60": 1.35111, - "61": 2.03462, - "62": 2.04452, - "63": 1.48275, - "64": 1.97048, - "65": 1.96045, - "66": 2.08338, - "67": 1.70836, - "68": 1.37627, - "69": 1.7677, - "70": 1.41652, - "71": 1.93616, - "72": 1.35719, - "73": 1.77487, - "74": 1.82048, - "75": 1.66406, - "76": 1.75487, - "77": 1.82277, - "78": 1.68683, - "79": 2.15529, - "80": 1.71822, - "81": 2.0064, - "82": 1.90058, - "83": 1.70485, - "84": 1.45853, - "85": 1.63833, - "86": 1.66802, - "87": 1.95107, - "88": 1.92734, - "89": 1.7966, - "90": 1.59094, - "91": 1.74259, - "92": 2.19268, - "93": 1.33438, - "94": 1.98423, - "95": 1.84764, - "96": 2.10523, - "97": 1.76641, - "98": 1.78903, - "99": 2.35571, - "100": 1.59126 + "2": 6.07428, + "3": 1.62298, + "4": 1.58419, + "5": 1.20999, + "6": 1.79668, + "7": 1.15658, + "8": 1.9972, + "9": 1.97455, + "10": 1.68015, + "11": 1.80729, + "12": 1.37982, + "13": 1.63794, + "14": 2.02358, + "15": 1.48981, + "16": 1.76656, + "17": 1.67224, + "18": 1.37557, + "19": 1.60636, + "20": 1.32815, + "21": 1.62961, + "22": 1.40272, + "23": 1.45115, + "24": 1.37151, + "25": 1.79749, + "26": 1.92617, + "27": 1.71135, + "28": 1.86339, + "29": 1.29876, + "30": 1.76434, + "31": 1.2512, + "32": 2.47535, + "33": 1.35886, + "34": 1.22059, + "35": 1.73808, + "36": 1.74589, + "37": 1.55778, + "38": 1.12072, + "39": 1.90363, + "40": 1.4491, + "41": 2.01462, + "42": 1.91276, + "43": 1.45822, + "44": 1.46029, + "45": 1.84333, + "46": 1.66872, + "47": 1.6508, + "48": 1.81958, + "49": 1.21417, + "50": 2.08162, + "51": 0.91474, + "52": 1.87095, + "53": 1.52191, + "54": 1.77984, + "55": 1.50287, + "56": 2.15344, + "57": 1.02955, + "58": 1.76077, + "59": 1.34124, + "60": 1.62024, + "61": 1.62154, + "62": 1.90407, + "63": 1.26589, + "64": 1.68437, + "65": 1.77885, + "66": 2.00597, + "67": 1.54811, + "68": 1.37648, + "69": 1.31429, + "70": 1.505, + "71": 2.06493, + "72": 1.30369, + "73": 1.69373, + "74": 1.25186, + "75": 1.66947, + "76": 1.72168, + "77": 1.40723, + "78": 1.41992, + "79": 2.39896, + "80": 1.74767, + "81": 1.77187, + "82": 2.06573, + "83": 1.41411, + "84": 1.45335, + "85": 1.50275, + "86": 1.44887, + "87": 1.93845, + "88": 1.62097, + "89": 2.10391, + "90": 1.72691, + "91": 1.56566, + "92": 1.86336, + "93": 1.69742, + "94": 1.67295, + "95": 1.62258, + "96": 1.78416, + "97": 1.17011, + "98": 1.77329, + "99": 2.15396, + "100": 1.67428 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json index fa8c625ca77..27ed4dd8cc2 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94183, - "2": 10.93668, - "3": 10.93421, - "4": 10.94284, - "5": 10.93215, - "6": 10.9333, - "7": 10.93826, - "8": 10.937, - "9": 10.93299, - "10": 10.93588, - "11": 10.92316, + "1": 10.94173, + "2": 10.9367, + "3": 10.93429, + "4": 10.94283, + "5": 10.93212, + "6": 10.93333, + "7": 10.93833, + "8": 10.93699, + "9": 10.93307, + "10": 10.93589, + "11": 10.92317, "12": 10.92245, - "13": 10.92252, - "14": 10.91553, - "15": 10.89066, - "16": 10.88926, + "13": 10.92264, + "14": 10.91546, + "15": 10.89074, + "16": 10.88932, "17": 10.89992, - "18": 10.87884, - "19": 10.87357, - "20": 10.79061, - "21": 10.78104, - "22": 10.77853, - "23": 10.77628, - "24": 10.73435, - "25": 10.74475, - "26": 10.72488, - "27": 10.69278, - "28": 10.62285, - "29": 10.60184, - "30": 10.58025, - "31": 10.56947, - "32": 10.54984, - "33": 10.52287, - "34": 10.48603, - "35": 10.49154, - "36": 10.47138, + "18": 10.8788, + "19": 10.87364, + "20": 10.7906, + "21": 10.78105, + "22": 10.77847, + "23": 10.77638, + "24": 10.73443, + "25": 10.74474, + "26": 10.72487, + "27": 10.6928, + "28": 10.62286, + "29": 10.60178, + "30": 10.58031, + "31": 10.56953, + "32": 10.54993, + "33": 10.52291, + "34": 10.48605, + "35": 10.49156, + "36": 10.47139, "37": 10.43631, - "38": 10.43821, - "39": 10.40192, - "40": 10.39027, - "41": 10.37045, - "42": 10.34738, - "43": 10.3245, - "44": 10.29041, - "45": 10.30667, - "46": 10.26823, - "47": 10.25077, - "48": 10.20225, + "38": 10.43828, + "39": 10.40197, + "40": 10.39031, + "41": 10.37054, + "42": 10.34744, + "43": 10.32452, + "44": 10.29042, + "45": 10.30675, + "46": 10.26825, + "47": 10.25083, + "48": 10.20234, "49": 10.20131, - "50": 10.21026, - "51": 10.20447, + "50": 10.21035, + "51": 10.20451, "52": 10.15575, - "53": 10.16786, - "54": 10.13054, - "55": 10.09996, - "56": 10.12943, - "57": 10.11355, - "58": 10.13081, - "59": 10.07364, - "60": 10.08689, + "53": 10.16794, + "54": 10.1306, + "55": 10.10005, + "56": 10.12947, + "57": 10.11361, + "58": 10.13083, + "59": 10.07368, + "60": 10.08694, "61": 10.04215, - "62": 10.01101, - "63": 10.0837, - "64": 10.03748, - "65": 10.009, - "66": 10.03921, + "62": 10.011, + "63": 10.08374, + "64": 10.03754, + "65": 10.00906, + "66": 10.03922, "67": 10.007, - "68": 9.97355, - "69": 9.99337, - "70": 9.98305, - "71": 10.00211, - "72": 9.97315, - "73": 9.96907, - "74": 9.95897, - "75": 9.92488, - "76": 9.96564, - "77": 9.95635, - "78": 9.90225, - "79": 9.91015, - "80": 9.92304, - "81": 9.94406, - "82": 9.88658, - "83": 9.84444, - "84": 9.7843, - "85": 9.77612, - "86": 9.87725, - "87": 9.91084, - "88": 9.88227, - "89": 9.82005, - "90": 9.80697, - "91": 9.81925, - "92": 9.80884, - "93": 9.74747, - "94": 9.82086, - "95": 9.81603, - "96": 9.80233, - "97": 9.74007, - "98": 9.76952, - "99": 9.816, - "100": 9.70637 + "68": 9.97362, + "69": 9.99338, + "70": 9.98309, + "71": 10.00214, + "72": 9.97318, + "73": 9.96916, + "74": 9.95901, + "75": 9.9249, + "76": 9.9657, + "77": 9.95644, + "78": 9.90233, + "79": 9.9102, + "80": 9.92303, + "81": 9.94407, + "82": 9.88666, + "83": 9.84451, + "84": 9.78438, + "85": 9.77617, + "86": 9.87733, + "87": 9.91081, + "88": 9.88238, + "89": 9.82008, + "90": 9.80707, + "91": 9.81929, + "92": 9.80893, + "93": 9.74751, + "94": 9.82092, + "95": 9.81609, + "96": 9.80235, + "97": 9.74012, + "98": 9.76957, + "99": 9.81601, + "100": 9.70638 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1932.0, - "2": 1782.0, - "3": 1862.0, - "4": 1715.0, - "5": 1742.0, + "1": 1834.0, + "2": 1767.0, + "3": 1788.0, + "4": 1755.0, + "5": 1755.0, "6": 1720.0, - "7": 2040.0, - "8": 1780.0, - "9": 1802.0, - "10": 1819.0, - "11": 1714.0, - "12": 1575.0, - "13": 1797.0, - "14": 1822.0, - "15": 1714.0, - "16": 1699.0, - "17": 1856.0, - "18": 1831.0, - "19": 1685.0, - "20": 1651.0, - "21": 1813.0, - "22": 1743.0, - "23": 1616.0, - "24": 1820.0, - "25": 1739.0, - "26": 1734.0, - "27": 1814.0, - "28": 1747.0, - "29": 2018.0, - "30": 1842.0, - "31": 2071.0, - "32": 1982.0, - "33": 2003.0, - "34": 1962.0, - "35": 2061.0, - "36": 1900.0, - "37": 2131.0, - "38": 2068.0, - "39": 2186.0, - "40": 2264.0, - "41": 2440.0, - "42": 2032.0, - "43": 2336.0, - "44": 2111.0, - "45": 2594.0, - "46": 2419.0, - "47": 2421.0, - "48": 2572.0, - "49": 2816.0, - "50": 2680.0, - "51": 2442.0, - "52": 2800.0, - "53": 2611.0, - "54": 2721.0, - "55": 2513.0, - "56": 2708.0, - "57": 2211.0, - "58": 3486.0, - "59": 2837.0, - "60": 2922.0, - "61": 2620.0, - "62": 3088.0, - "63": 3165.0, - "64": 3436.0, - "65": 2650.0, - "66": 3051.0, - "67": 3859.0, - "68": 3358.0, - "69": 2980.0, - "70": 3076.0, - "71": 3004.0, - "72": 2978.0, - "73": 3368.0, - "74": 3263.0, - "75": 3164.0, - "76": 3059.0, - "77": 3850.0, - "78": 3163.0, - "79": 3182.0, - "80": 2935.0, - "81": 3344.0, - "82": 3107.0, - "83": 3147.0, - "84": 2982.0, - "85": 2804.0, - "86": 3000.0, - "87": 2971.0, - "88": 2995.0, - "89": 2875.0, - "90": 3463.0, - "91": 2995.0, - "92": 2874.0, - "93": 3072.0, - "94": 3289.0, - "95": 3495.0, - "96": 3436.0, - "97": 3655.0, - "98": 3577.0, - "99": 3220.0, - "100": 3266.0 + "7": 2043.0, + "8": 1813.0, + "9": 1774.0, + "10": 1763.0, + "11": 1660.0, + "12": 1742.0, + "13": 1915.0, + "14": 1861.0, + "15": 1735.0, + "16": 1768.0, + "17": 1819.0, + "18": 1844.0, + "19": 1703.0, + "20": 1684.0, + "21": 1824.0, + "22": 1769.0, + "23": 1644.0, + "24": 1829.0, + "25": 1828.0, + "26": 1830.0, + "27": 1809.0, + "28": 1895.0, + "29": 1998.0, + "30": 1885.0, + "31": 2032.0, + "32": 1986.0, + "33": 1886.0, + "34": 2082.0, + "35": 2001.0, + "36": 1941.0, + "37": 2223.0, + "38": 2103.0, + "39": 2084.0, + "40": 2271.0, + "41": 2430.0, + "42": 1965.0, + "43": 2332.0, + "44": 2214.0, + "45": 2533.0, + "46": 2344.0, + "47": 2433.0, + "48": 2597.0, + "49": 2801.0, + "50": 2575.0, + "51": 2428.0, + "52": 2677.0, + "53": 2509.0, + "54": 2697.0, + "55": 2491.0, + "56": 2635.0, + "57": 2268.0, + "58": 3560.0, + "59": 2877.0, + "60": 2886.0, + "61": 2768.0, + "62": 3054.0, + "63": 3190.0, + "64": 3405.0, + "65": 2706.0, + "66": 3055.0, + "67": 3819.0, + "68": 3349.0, + "69": 2909.0, + "70": 3134.0, + "71": 3052.0, + "72": 2847.0, + "73": 3339.0, + "74": 3192.0, + "75": 3154.0, + "76": 3203.0, + "77": 3788.0, + "78": 3181.0, + "79": 3210.0, + "80": 3063.0, + "81": 3235.0, + "82": 3037.0, + "83": 3031.0, + "84": 2879.0, + "85": 2694.0, + "86": 3096.0, + "87": 2850.0, + "88": 3050.0, + "89": 2886.0, + "90": 3551.0, + "91": 2902.0, + "92": 2889.0, + "93": 3085.0, + "94": 3204.0, + "95": 3429.0, + "96": 3384.0, + "97": 3614.0, + "98": 3588.0, + "99": 3221.0, + "100": 3279.0 } }, "mem-allocated-bytes": { @@ -325,7 +325,7 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2397240320.0, + "1": 2398288384.0, "2": 2681049088.0, "3": 2681049088.0, "4": 2681049088.0, @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.03808, - "3": 0.12679, - "4": 0.10946, - "5": 0.10855, - "6": 0.11047, - "7": 0.11168, - "8": 0.11197, - "9": 0.11246, - "10": 0.11206, - "11": 0.11105, - "12": 0.11105, - "13": 0.11147, - "14": 0.11138, - "15": 0.11228, - "16": 0.11223, - "17": 0.11293, - "18": 0.11304, - "19": 0.11272, - "20": 0.1118, - "21": 0.11175, - "22": 0.11203, - "23": 0.11243, - "24": 0.11233, - "25": 0.1122, - "26": 0.11267, - "27": 0.11236, - "28": 0.11188, - "29": 0.11147, - "30": 0.11142, - "31": 0.11223, - "32": 0.11266, - "33": 0.11134, - "34": 0.11125, - "35": 0.11183, - "36": 0.11284, - "37": 0.11337, - "38": 0.11223, - "39": 0.11206, - "40": 0.11313, - "41": 0.11284, - "42": 0.11385, - "43": 0.11233, - "44": 0.11129, - "45": 0.11354, - "46": 0.11208, - "47": 0.11221, - "48": 0.11374, - "49": 0.11227, - "50": 0.11298, - "51": 0.27153, - "52": 0.15025, - "53": 0.11506, - "54": 0.11374, - "55": 0.11174, - "56": 0.11275, - "57": 0.11229, - "58": 0.11369, - "59": 0.11289, - "60": 0.11313, - "61": 0.113, - "62": 0.11197, - "63": 0.11485, - "64": 0.11249, - "65": 0.11353, - "66": 0.11378, - "67": 0.1127, - "68": 0.11211, - "69": 0.1132, - "70": 0.11291, - "71": 0.1138, - "72": 0.11551, - "73": 0.11471, - "74": 0.11572, - "75": 0.1155, - "76": 0.11598, - "77": 0.11562, - "78": 0.11521, - "79": 0.11708, - "80": 0.11616, - "81": 0.11524, - "82": 0.11546, - "83": 0.11785, - "84": 0.11661, - "85": 0.11642, - "86": 0.1175, - "87": 0.11787, - "88": 0.11551, - "89": 0.11855, - "90": 0.11723, - "91": 0.11733, - "92": 0.11855, - "93": 0.11835, - "94": 0.11528, - "95": 0.11504, - "96": 0.11521, - "97": 0.11531, - "98": 0.11548, - "99": 0.1139, - "100": 0.11411 + "2": 11.18471, + "3": 0.1456, + "4": 0.1008, + "5": 0.09802, + "6": 0.09552, + "7": 0.09801, + "8": 0.09442, + "9": 0.09856, + "10": 0.09743, + "11": 0.09791, + "12": 0.10005, + "13": 0.09814, + "14": 0.09986, + "15": 0.09852, + "16": 0.09808, + "17": 0.10185, + "18": 0.10092, + "19": 0.09692, + "20": 0.10136, + "21": 0.09843, + "22": 0.1046, + "23": 0.10414, + "24": 0.09875, + "25": 0.0997, + "26": 0.0989, + "27": 0.10031, + "28": 0.10055, + "29": 0.10027, + "30": 0.1006, + "31": 0.09985, + "32": 0.10071, + "33": 0.09783, + "34": 0.09907, + "35": 0.09971, + "36": 0.09869, + "37": 0.10135, + "38": 0.097, + "39": 0.09775, + "40": 0.09805, + "41": 0.09874, + "42": 0.09833, + "43": 0.09679, + "44": 0.09734, + "45": 0.09819, + "46": 0.0981, + "47": 0.09848, + "48": 0.10098, + "49": 0.09913, + "50": 0.09955, + "51": 0.24081, + "52": 0.15008, + "53": 0.10996, + "54": 0.10225, + "55": 0.09888, + "56": 0.1047, + "57": 0.10529, + "58": 0.10246, + "59": 0.10097, + "60": 0.10045, + "61": 0.10033, + "62": 0.10271, + "63": 0.09934, + "64": 0.10053, + "65": 0.10051, + "66": 0.10197, + "67": 0.09938, + "68": 0.09764, + "69": 0.09682, + "70": 0.0998, + "71": 0.09798, + "72": 0.09904, + "73": 0.09968, + "74": 0.09882, + "75": 0.09792, + "76": 0.10009, + "77": 0.10405, + "78": 0.09866, + "79": 0.09961, + "80": 0.09845, + "81": 0.09759, + "82": 0.09629, + "83": 0.09695, + "84": 0.0964, + "85": 0.09973, + "86": 0.09733, + "87": 0.09663, + "88": 0.09831, + "89": 0.0968, + "90": 0.09435, + "91": 0.09681, + "92": 0.09496, + "93": 0.13849, + "94": 0.09714, + "95": 0.09772, + "96": 0.09792, + "97": 0.09944, + "98": 0.09856, + "99": 0.09891, + "100": 0.09901 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json index 8040aa62289..cb10a675000 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95854, - "4": 10.95197, - "5": 10.95261, + "1": 10.96462, + "2": 10.95232, + "3": 10.95859, + "4": 10.95192, + "5": 10.95262, "6": 10.95382, - "7": 10.95159, - "8": 10.95132, - "9": 10.9587, - "10": 10.94456, - "11": 10.93683, - "12": 10.94343, - "13": 10.94314, - "14": 10.94294, - "15": 10.91497, - "16": 10.90029, - "17": 10.9111, - "18": 10.90623, - "19": 10.90165, - "20": 10.80789, - "21": 10.79458, - "22": 10.80475, - "23": 10.78977, - "24": 10.77497, - "25": 10.76167, + "7": 10.95158, + "8": 10.95135, + "9": 10.95869, + "10": 10.9445, + "11": 10.93682, + "12": 10.94337, + "13": 10.94313, + "14": 10.94293, + "15": 10.91492, + "16": 10.90026, + "17": 10.91109, + "18": 10.90612, + "19": 10.90162, + "20": 10.80786, + "21": 10.79455, + "22": 10.80479, + "23": 10.78975, + "24": 10.77506, + "25": 10.76169, "26": 10.75516, - "27": 10.71956, - "28": 10.63913, - "29": 10.60851, - "30": 10.58649, - "31": 10.59096, - "32": 10.57108, - "33": 10.54031, - "34": 10.49895, - "35": 10.50028, - "36": 10.48631, - "37": 10.44931, - "38": 10.4507, - "39": 10.41767, - "40": 10.39538, - "41": 10.37482, - "42": 10.3606, - "43": 10.33045, - "44": 10.31901, - "45": 10.31602, - "46": 10.27652, - "47": 10.26823, - "48": 10.21834, - "49": 10.21388, - "50": 10.22196, - "51": 10.22216, - "52": 10.16751, - "53": 10.17223, - "54": 10.13795, - "55": 10.10895, + "27": 10.71953, + "28": 10.63907, + "29": 10.60855, + "30": 10.58653, + "31": 10.59094, + "32": 10.57111, + "33": 10.54035, + "34": 10.49898, + "35": 10.50036, + "36": 10.4863, + "37": 10.44938, + "38": 10.45068, + "39": 10.41773, + "40": 10.39546, + "41": 10.37489, + "42": 10.36063, + "43": 10.33047, + "44": 10.31911, + "45": 10.31612, + "46": 10.27656, + "47": 10.2683, + "48": 10.21835, + "49": 10.21389, + "50": 10.22203, + "51": 10.22221, + "52": 10.16757, + "53": 10.17229, + "54": 10.138, + "55": 10.10898, "56": 10.13223, - "57": 10.1214, + "57": 10.12137, "58": 10.12635, - "59": 10.07039, - "60": 10.09306, - "61": 10.04907, + "59": 10.07045, + "60": 10.0931, + "61": 10.04912, "62": 10.01778, - "63": 10.08391, - "64": 10.03345, - "65": 10.00557, - "66": 10.04739, - "67": 10.01996, - "68": 9.98697, - "69": 9.99635, - "70": 9.98279, - "71": 10.00545, - "72": 9.99249, - "73": 9.97618, - "74": 9.96756, - "75": 9.93703, - "76": 9.96635, - "77": 9.96325, - "78": 9.91485, - "79": 9.91547, - "80": 9.92964, - "81": 9.9558, - "82": 9.89447, - "83": 9.85962, - "84": 9.80306, - "85": 9.78431, - "86": 9.88636, - "87": 9.90914, - "88": 9.88157, - "89": 9.82173, - "90": 9.82085, - "91": 9.82114, - "92": 9.81671, - "93": 9.74868, + "63": 10.08401, + "64": 10.03352, + "65": 10.00559, + "66": 10.04745, + "67": 10.02001, + "68": 9.98705, + "69": 9.99636, + "70": 9.98282, + "71": 10.0055, + "72": 9.99252, + "73": 9.97621, + "74": 9.9676, + "75": 9.93709, + "76": 9.96638, + "77": 9.96329, + "78": 9.91487, + "79": 9.9155, + "80": 9.92968, + "81": 9.95591, + "82": 9.89454, + "83": 9.85968, + "84": 9.8031, + "85": 9.78436, + "86": 9.88639, + "87": 9.90911, + "88": 9.8816, + "89": 9.8218, + "90": 9.82094, + "91": 9.82118, + "92": 9.81674, + "93": 9.7487, "94": 9.82364, - "95": 9.811, - "96": 9.79716, - "97": 9.74559, - "98": 9.7656, - "99": 9.82203, - "100": 9.70436 + "95": 9.81102, + "96": 9.7972, + "97": 9.74564, + "98": 9.76566, + "99": 9.82205, + "100": 9.70444 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1787.0, - "4": 1675.0, - "5": 1820.0, - "6": 1678.0, - "7": 1861.0, - "8": 1749.0, - "9": 1745.0, - "10": 1815.0, - "11": 1687.0, - "12": 1746.0, - "13": 1767.0, - "14": 1933.0, - "15": 1608.0, - "16": 1731.0, - "17": 1843.0, - "18": 1811.0, - "19": 1698.0, - "20": 1761.0, - "21": 1798.0, - "22": 1807.0, - "23": 1700.0, - "24": 1838.0, - "25": 1714.0, - "26": 1779.0, - "27": 1769.0, - "28": 1784.0, - "29": 1850.0, - "30": 1800.0, - "31": 2098.0, - "32": 1919.0, - "33": 1958.0, - "34": 1939.0, - "35": 2002.0, - "36": 1970.0, - "37": 2256.0, - "38": 2144.0, - "39": 2190.0, - "40": 2230.0, - "41": 2287.0, - "42": 2018.0, - "43": 2414.0, - "44": 2132.0, - "45": 2497.0, - "46": 2470.0, - "47": 2384.0, - "48": 2553.0, - "49": 2756.0, - "50": 2473.0, - "51": 2360.0, - "52": 2657.0, - "53": 2568.0, - "54": 2665.0, - "55": 2431.0, - "56": 2636.0, - "57": 2269.0, - "58": 3524.0, - "59": 2901.0, - "60": 2907.0, - "61": 2656.0, - "62": 3131.0, - "63": 3144.0, - "64": 3566.0, - "65": 2764.0, - "66": 3027.0, - "67": 3535.0, - "68": 3272.0, - "69": 2912.0, - "70": 3362.0, - "71": 3120.0, - "72": 2865.0, - "73": 3395.0, - "74": 3241.0, - "75": 3118.0, - "76": 3261.0, - "77": 3678.0, - "78": 3193.0, - "79": 3321.0, - "80": 3038.0, - "81": 3427.0, - "82": 2976.0, - "83": 3100.0, - "84": 2986.0, - "85": 2684.0, - "86": 3033.0, - "87": 2888.0, - "88": 3016.0, - "89": 3192.0, - "90": 3783.0, - "91": 3055.0, - "92": 3053.0, - "93": 3151.0, - "94": 2886.0, - "95": 3384.0, - "96": 3437.0, - "97": 3594.0, - "98": 3294.0, - "99": 3050.0, - "100": 3147.0 + "1": 1759.0, + "2": 1644.0, + "3": 1736.0, + "4": 1783.0, + "5": 1782.0, + "6": 1759.0, + "7": 1975.0, + "8": 1703.0, + "9": 1743.0, + "10": 1740.0, + "11": 1686.0, + "12": 1707.0, + "13": 1844.0, + "14": 1792.0, + "15": 1617.0, + "16": 1755.0, + "17": 1772.0, + "18": 1850.0, + "19": 1711.0, + "20": 1735.0, + "21": 1832.0, + "22": 1710.0, + "23": 1745.0, + "24": 1863.0, + "25": 1685.0, + "26": 1782.0, + "27": 1728.0, + "28": 1716.0, + "29": 1901.0, + "30": 1867.0, + "31": 2036.0, + "32": 2025.0, + "33": 1906.0, + "34": 1928.0, + "35": 2040.0, + "36": 1924.0, + "37": 2258.0, + "38": 2111.0, + "39": 2127.0, + "40": 2234.0, + "41": 2314.0, + "42": 1988.0, + "43": 2306.0, + "44": 2077.0, + "45": 2420.0, + "46": 2388.0, + "47": 2346.0, + "48": 2653.0, + "49": 2773.0, + "50": 2409.0, + "51": 2356.0, + "52": 2637.0, + "53": 2611.0, + "54": 2635.0, + "55": 2533.0, + "56": 2604.0, + "57": 2235.0, + "58": 3558.0, + "59": 2899.0, + "60": 2953.0, + "61": 2699.0, + "62": 3095.0, + "63": 3265.0, + "64": 3535.0, + "65": 2783.0, + "66": 3039.0, + "67": 3734.0, + "68": 3381.0, + "69": 2857.0, + "70": 3468.0, + "71": 3162.0, + "72": 2982.0, + "73": 3439.0, + "74": 3386.0, + "75": 3136.0, + "76": 3295.0, + "77": 3595.0, + "78": 3263.0, + "79": 3243.0, + "80": 3081.0, + "81": 3378.0, + "82": 2984.0, + "83": 2960.0, + "84": 2966.0, + "85": 2778.0, + "86": 3096.0, + "87": 2789.0, + "88": 3109.0, + "89": 3222.0, + "90": 3747.0, + "91": 2964.0, + "92": 3082.0, + "93": 3211.0, + "94": 2881.0, + "95": 3408.0, + "96": 3439.0, + "97": 3412.0, + "98": 3441.0, + "99": 3046.0, + "100": 3117.0 } }, "mem-allocated-bytes": { @@ -242,7 +242,7 @@ "22": 766367232.0, "23": 766367232.0, "24": 766367232.0, - "25": 766367232.0, + "25": 767415808.0, "26": 766367232.0, "27": 766367232.0, "28": 766367232.0, @@ -377,54 +377,54 @@ "50": 2647494656.0, "51": 2647494656.0, "52": 2647494656.0, - "53": 2647494656.0, - "54": 2647494656.0, - "55": 2647494656.0, - "56": 2647494656.0, - "57": 2647494656.0, - "58": 2647494656.0, - "59": 2647494656.0, - "60": 2647494656.0, - "61": 2647494656.0, - "62": 2647494656.0, - "63": 2647494656.0, - "64": 2647494656.0, - "65": 2647494656.0, - "66": 2647494656.0, - "67": 2647494656.0, - "68": 2647494656.0, - "69": 2647494656.0, - "70": 2647494656.0, - "71": 2647494656.0, - "72": 2647494656.0, - "73": 2647494656.0, - "74": 2647494656.0, - "75": 2647494656.0, - "76": 2647494656.0, - "77": 2647494656.0, - "78": 2647494656.0, - "79": 2647494656.0, - "80": 2647494656.0, - "81": 2647494656.0, - "82": 2647494656.0, - "83": 2647494656.0, - "84": 2647494656.0, - "85": 2647494656.0, - "86": 2647494656.0, - "87": 2647494656.0, - "88": 2647494656.0, - "89": 2647494656.0, - "90": 2647494656.0, - "91": 2647494656.0, - "92": 2647494656.0, - "93": 2647494656.0, - "94": 2647494656.0, - "95": 2647494656.0, - "96": 2647494656.0, - "97": 2647494656.0, - "98": 2647494656.0, - "99": 2647494656.0, - "100": 2647494656.0 + "53": 2648412160.0, + "54": 2648412160.0, + "55": 2648412160.0, + "56": 2648412160.0, + "57": 2648412160.0, + "58": 2648412160.0, + "59": 2648412160.0, + "60": 2648412160.0, + "61": 2648412160.0, + "62": 2648412160.0, + "63": 2648412160.0, + "64": 2648412160.0, + "65": 2648412160.0, + "66": 2648412160.0, + "67": 2648412160.0, + "68": 2648412160.0, + "69": 2648412160.0, + "70": 2648412160.0, + "71": 2648412160.0, + "72": 2648412160.0, + "73": 2648412160.0, + "74": 2648412160.0, + "75": 2648412160.0, + "76": 2648412160.0, + "77": 2648412160.0, + "78": 2648412160.0, + "79": 2648412160.0, + "80": 2648412160.0, + "81": 2648412160.0, + "82": 2648412160.0, + "83": 2648412160.0, + "84": 2648412160.0, + "85": 2648412160.0, + "86": 2648412160.0, + "87": 2648412160.0, + "88": 2648412160.0, + "89": 2648412160.0, + "90": 2648412160.0, + "91": 2648412160.0, + "92": 2648412160.0, + "93": 2648412160.0, + "94": 2648412160.0, + "95": 2648412160.0, + "96": 2648412160.0, + "97": 2648412160.0, + "98": 2648412160.0, + "99": 2648412160.0, + "100": 2648412160.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.07645, - "3": 0.1094, - "4": 0.10987, - "5": 0.10883, - "6": 0.11143, - "7": 0.11116, - "8": 0.10899, - "9": 0.10822, - "10": 0.10745, - "11": 0.11042, - "12": 0.11135, - "13": 0.1125, - "14": 0.10741, - "15": 0.10975, - "16": 0.11197, - "17": 0.10864, - "18": 0.11007, - "19": 0.10982, - "20": 0.1076, - "21": 0.10764, - "22": 0.10691, - "23": 0.10504, - "24": 0.10802, - "25": 0.1067, - "26": 0.10698, - "27": 0.1106, - "28": 0.10955, - "29": 0.10759, - "30": 0.11102, - "31": 0.10646, - "32": 0.1044, - "33": 0.10659, - "34": 0.10918, - "35": 0.10885, - "36": 0.10532, - "37": 0.11209, - "38": 0.1102, - "39": 0.10672, - "40": 0.10605, - "41": 0.106, - "42": 0.10788, - "43": 0.1076, - "44": 0.1109, - "45": 0.10762, - "46": 0.1075, - "47": 0.10575, - "48": 0.10826, - "49": 0.1088, - "50": 0.10673, - "51": 0.14127, - "52": 0.12881, - "53": 0.10771, - "54": 0.11168, - "55": 0.10677, - "56": 0.10627, - "57": 0.10897, - "58": 0.1109, - "59": 0.115, - "60": 0.10854, - "61": 0.11013, - "62": 0.11119, - "63": 0.11036, - "64": 0.10763, - "65": 0.11107, - "66": 0.10841, - "67": 0.10371, - "68": 0.10647, - "69": 0.10758, - "70": 0.10966, - "71": 0.10927, - "72": 0.10884, - "73": 0.10782, - "74": 0.10843, - "75": 0.1068, - "76": 0.10932, - "77": 0.1077, - "78": 0.11001, - "79": 0.10886, - "80": 0.10761, - "81": 0.10634, - "82": 0.10517, - "83": 0.11168, - "84": 0.10628, - "85": 0.11065, - "86": 0.10353, - "87": 0.10602, - "88": 0.11046, - "89": 0.10846, - "90": 0.10902, - "91": 0.10778, - "92": 0.10978, - "93": 0.10922, - "94": 0.10581, - "95": 0.10786, - "96": 0.10787, - "97": 0.10592, - "98": 0.10606, - "99": 0.10961, - "100": 0.10901 + "2": 7.66756, + "3": 0.11005, + "4": 0.11326, + "5": 0.10905, + "6": 0.10784, + "7": 0.10904, + "8": 0.10648, + "9": 0.10981, + "10": 0.10945, + "11": 0.11262, + "12": 0.11099, + "13": 0.11037, + "14": 0.10692, + "15": 0.10707, + "16": 0.10732, + "17": 0.11319, + "18": 0.10897, + "19": 0.10966, + "20": 0.11075, + "21": 0.10958, + "22": 0.1104, + "23": 0.11723, + "24": 0.11162, + "25": 0.106, + "26": 0.1074, + "27": 0.10991, + "28": 0.10995, + "29": 0.10928, + "30": 0.10978, + "31": 0.10823, + "32": 0.10849, + "33": 0.10619, + "34": 0.10616, + "35": 0.1062, + "36": 0.10801, + "37": 0.10994, + "38": 0.10857, + "39": 0.10958, + "40": 0.10688, + "41": 0.10922, + "42": 0.10724, + "43": 0.10813, + "44": 0.11179, + "45": 0.11268, + "46": 0.10756, + "47": 0.11356, + "48": 0.11074, + "49": 0.11119, + "50": 0.10841, + "51": 0.14959, + "52": 0.1385, + "53": 0.11791, + "54": 0.11672, + "55": 0.1152, + "56": 0.11299, + "57": 0.11792, + "58": 0.11451, + "59": 0.11599, + "60": 0.1158, + "61": 0.11681, + "62": 0.11573, + "63": 0.11691, + "64": 0.11503, + "65": 0.11263, + "66": 0.11281, + "67": 0.1156, + "68": 0.11651, + "69": 0.11511, + "70": 0.11291, + "71": 0.11337, + "72": 0.11552, + "73": 0.1175, + "74": 0.11475, + "75": 0.11566, + "76": 0.11437, + "77": 0.1143, + "78": 0.11383, + "79": 0.11633, + "80": 0.1183, + "81": 0.11383, + "82": 0.11578, + "83": 0.11332, + "84": 0.11395, + "85": 0.11314, + "86": 0.10983, + "87": 0.11393, + "88": 0.11639, + "89": 0.1149, + "90": 0.11534, + "91": 0.11567, + "92": 0.11559, + "93": 0.11858, + "94": 0.11765, + "95": 0.11654, + "96": 0.11902, + "97": 0.11699, + "98": 0.11194, + "99": 0.11311, + "100": 0.11617 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json index 68a163b9bd4..560a9f7794d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.9423, - "5": 10.93352, - "6": 10.93273, - "7": 10.93793, - "8": 10.93357, - "9": 10.93552, - "10": 10.93693, - "11": 10.92585, - "12": 10.92413, - "13": 10.9251, - "14": 10.91258, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.94226, + "5": 10.93351, + "6": 10.93269, + "7": 10.93786, + "8": 10.93354, + "9": 10.93555, + "10": 10.93689, + "11": 10.92582, + "12": 10.92411, + "13": 10.92513, + "14": 10.91252, "15": 10.89335, - "16": 10.88822, - "17": 10.89973, - "18": 10.87821, + "16": 10.8883, + "17": 10.89968, + "18": 10.87816, "19": 10.87589, - "20": 10.79119, + "20": 10.79127, "21": 10.78019, - "22": 10.77474, - "23": 10.77501, - "24": 10.7364, - "25": 10.74383, - "26": 10.72466, - "27": 10.69626, - "28": 10.62353, - "29": 10.60379, - "30": 10.58048, + "22": 10.77477, + "23": 10.77506, + "24": 10.73639, + "25": 10.74386, + "26": 10.72465, + "27": 10.69633, + "28": 10.62354, + "29": 10.60385, + "30": 10.58047, "31": 10.56959, "32": 10.5486, - "33": 10.52046, - "34": 10.48553, - "35": 10.4923, - "36": 10.47086, - "37": 10.43635, - "38": 10.44109, - "39": 10.40281, - "40": 10.3928, - "41": 10.36862, - "42": 10.34609, - "43": 10.32634, - "44": 10.28964, - "45": 10.3083, - "46": 10.2682, - "47": 10.2525, - "48": 10.20372, - "49": 10.20119, - "50": 10.20922, - "51": 10.20528, - "52": 10.15731, + "33": 10.52062, + "34": 10.48556, + "35": 10.49231, + "36": 10.47101, + "37": 10.43633, + "38": 10.44118, + "39": 10.40284, + "40": 10.39291, + "41": 10.36868, + "42": 10.34616, + "43": 10.32631, + "44": 10.28969, + "45": 10.30834, + "46": 10.26826, + "47": 10.25255, + "48": 10.20375, + "49": 10.20125, + "50": 10.20932, + "51": 10.20534, + "52": 10.15739, "53": 10.16687, - "54": 10.13043, - "55": 10.10057, + "54": 10.1305, + "55": 10.10061, "56": 10.13013, - "57": 10.11459, - "58": 10.13004, - "59": 10.07594, - "60": 10.08697, - "61": 10.04225, - "62": 10.00953, - "63": 10.08368, - "64": 10.03719, - "65": 10.00999, - "66": 10.04021, - "67": 10.00686, - "68": 9.97346, - "69": 9.99438, - "70": 9.98228, - "71": 10.00183, - "72": 9.97365, - "73": 9.96698, - "74": 9.95724, - "75": 9.92479, - "76": 9.96623, + "57": 10.11465, + "58": 10.13007, + "59": 10.07595, + "60": 10.08702, + "61": 10.0423, + "62": 10.00957, + "63": 10.08371, + "64": 10.03718, + "65": 10.01, + "66": 10.04025, + "67": 10.00692, + "68": 9.97351, + "69": 9.99448, + "70": 9.98227, + "71": 10.00189, + "72": 9.97374, + "73": 9.96701, + "74": 9.95731, + "75": 9.92482, + "76": 9.96625, "77": 9.95576, - "78": 9.90172, - "79": 9.90899, - "80": 9.92107, - "81": 9.94317, + "78": 9.90177, + "79": 9.90906, + "80": 9.92113, + "81": 9.94323, "82": 9.88583, - "83": 9.84493, - "84": 9.7831, - "85": 9.77443, - "86": 9.87683, - "87": 9.91001, - "88": 9.882, + "83": 9.845, + "84": 9.78317, + "85": 9.77446, + "86": 9.87691, + "87": 9.91006, + "88": 9.88207, "89": 9.81985, - "90": 9.80796, - "91": 9.81902, - "92": 9.80779, - "93": 9.74751, - "94": 9.82029, - "95": 9.8157, - "96": 9.80356, - "97": 9.7406, - "98": 9.76894, - "99": 9.81594, - "100": 9.70555 + "90": 9.80799, + "91": 9.81907, + "92": 9.80784, + "93": 9.74759, + "94": 9.82035, + "95": 9.81578, + "96": 9.80362, + "97": 9.74067, + "98": 9.76896, + "99": 9.81595, + "100": 9.7056 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1721.0, - "4": 1751.0, - "5": 1756.0, - "6": 1766.0, - "7": 2096.0, - "8": 1769.0, - "9": 1799.0, - "10": 1745.0, - "11": 1650.0, - "12": 1774.0, - "13": 1839.0, - "14": 1831.0, - "15": 1653.0, - "16": 1694.0, - "17": 1777.0, - "18": 1814.0, - "19": 1660.0, - "20": 1774.0, - "21": 1735.0, - "22": 1698.0, - "23": 1720.0, - "24": 1857.0, - "25": 1696.0, - "26": 1895.0, - "27": 1727.0, - "28": 1790.0, - "29": 1978.0, - "30": 1883.0, - "31": 2011.0, - "32": 2009.0, - "33": 2065.0, - "34": 2005.0, - "35": 1998.0, - "36": 1949.0, - "37": 2233.0, - "38": 2134.0, - "39": 2029.0, - "40": 2320.0, - "41": 2310.0, - "42": 2012.0, - "43": 2229.0, - "44": 2253.0, - "45": 2455.0, - "46": 2413.0, - "47": 2499.0, - "48": 2654.0, - "49": 2695.0, - "50": 2556.0, - "51": 2497.0, - "52": 2697.0, + "1": 1791.0, + "2": 1794.0, + "3": 1774.0, + "4": 1707.0, + "5": 1776.0, + "6": 1774.0, + "7": 2058.0, + "8": 1681.0, + "9": 1838.0, + "10": 1766.0, + "11": 1778.0, + "12": 1661.0, + "13": 1915.0, + "14": 1858.0, + "15": 1693.0, + "16": 1659.0, + "17": 1816.0, + "18": 1875.0, + "19": 1664.0, + "20": 1712.0, + "21": 1799.0, + "22": 1673.0, + "23": 1716.0, + "24": 1890.0, + "25": 1735.0, + "26": 1871.0, + "27": 1782.0, + "28": 1841.0, + "29": 1861.0, + "30": 1950.0, + "31": 1975.0, + "32": 1915.0, + "33": 1943.0, + "34": 1992.0, + "35": 2041.0, + "36": 1911.0, + "37": 2255.0, + "38": 2085.0, + "39": 2135.0, + "40": 2250.0, + "41": 2304.0, + "42": 1952.0, + "43": 2355.0, + "44": 2183.0, + "45": 2496.0, + "46": 2410.0, + "47": 2384.0, + "48": 2587.0, + "49": 2671.0, + "50": 2538.0, + "51": 2466.0, + "52": 2723.0, "53": 2566.0, - "54": 2737.0, - "55": 2430.0, - "56": 2681.0, - "57": 2235.0, - "58": 3500.0, - "59": 2810.0, - "60": 2847.0, - "61": 2634.0, - "62": 3234.0, - "63": 3180.0, - "64": 3344.0, - "65": 2659.0, - "66": 2923.0, - "67": 3727.0, - "68": 3419.0, - "69": 2980.0, - "70": 3155.0, - "71": 3102.0, - "72": 2985.0, - "73": 3265.0, - "74": 3183.0, - "75": 3040.0, - "76": 3010.0, - "77": 3807.0, - "78": 3133.0, - "79": 3258.0, - "80": 3010.0, - "81": 3341.0, - "82": 3261.0, - "83": 3212.0, - "84": 2927.0, - "85": 2838.0, - "86": 3179.0, - "87": 2957.0, - "88": 3042.0, - "89": 2842.0, - "90": 3453.0, - "91": 3015.0, - "92": 3062.0, - "93": 3085.0, - "94": 3098.0, - "95": 3397.0, - "96": 3546.0, - "97": 3660.0, - "98": 3603.0, - "99": 3169.0, - "100": 3271.0 + "54": 2789.0, + "55": 2483.0, + "56": 2606.0, + "57": 2206.0, + "58": 3501.0, + "59": 2982.0, + "60": 2905.0, + "61": 2699.0, + "62": 3139.0, + "63": 3239.0, + "64": 3365.0, + "65": 2646.0, + "66": 3002.0, + "67": 3724.0, + "68": 3388.0, + "69": 2901.0, + "70": 3363.0, + "71": 3001.0, + "72": 2955.0, + "73": 3332.0, + "74": 3180.0, + "75": 3066.0, + "76": 3097.0, + "77": 3692.0, + "78": 3079.0, + "79": 3113.0, + "80": 3031.0, + "81": 3333.0, + "82": 3182.0, + "83": 3173.0, + "84": 2896.0, + "85": 2796.0, + "86": 3091.0, + "87": 2911.0, + "88": 3087.0, + "89": 2758.0, + "90": 3507.0, + "91": 3127.0, + "92": 2971.0, + "93": 3166.0, + "94": 3124.0, + "95": 3452.0, + "96": 3566.0, + "97": 3541.0, + "98": 3564.0, + "99": 3267.0, + "100": 3287.0 } }, "mem-allocated-bytes": { @@ -283,7 +283,7 @@ "63": 764269056.0, "64": 764269056.0, "65": 764269056.0, - "66": 765317632.0, + "66": 764269056.0, "67": 764269056.0, "68": 764269056.0, "69": 764269056.0, @@ -291,7 +291,7 @@ "71": 764269056.0, "72": 764269056.0, "73": 764269056.0, - "74": 765317632.0, + "74": 764269056.0, "75": 764269056.0, "76": 764269056.0, "77": 764269056.0, @@ -303,7 +303,7 @@ "83": 764269056.0, "84": 764269056.0, "85": 764269056.0, - "86": 764269056.0, + "86": 765317632.0, "87": 764269056.0, "88": 764269056.0, "89": 764269056.0, @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2398288384.0, - "2": 2683147264.0, - "3": 2683147264.0, - "4": 2683671552.0, - "5": 2683671552.0, - "6": 2683671552.0, - "7": 2683671552.0, - "8": 2683671552.0, - "9": 2683671552.0, - "10": 2683671552.0, - "11": 2683671552.0, - "12": 2683671552.0, - "13": 2683671552.0, - "14": 2683671552.0, - "15": 2683671552.0, - "16": 2683671552.0, - "17": 2683671552.0, - "18": 2683671552.0, - "19": 2683671552.0, - "20": 2683671552.0, - "21": 2683671552.0, - "22": 2683671552.0, - "23": 2683671552.0, - "24": 2684456448.0, - "25": 2684456448.0, - "26": 2684456448.0, - "27": 2684456448.0, - "28": 2684456448.0, - "29": 2684456448.0, - "30": 2684456448.0, - "31": 2684456448.0, - "32": 2684456448.0, - "33": 2684456448.0, - "34": 2684456448.0, - "35": 2684456448.0, - "36": 2684456448.0, - "37": 2684456448.0, - "38": 2684456448.0, - "39": 2684456448.0, - "40": 2684456448.0, - "41": 2684456448.0, - "42": 2684456448.0, - "43": 2684456448.0, - "44": 2684456448.0, - "45": 2684456448.0, - "46": 2684456448.0, - "47": 2684456448.0, - "48": 2684456448.0, - "49": 2684456448.0, - "50": 2684456448.0, - "51": 2684456448.0, - "52": 2684456448.0, - "53": 2684456448.0, - "54": 2684456448.0, - "55": 2684456448.0, - "56": 2684456448.0, - "57": 2684456448.0, - "58": 2684456448.0, - "59": 2684456448.0, - "60": 2684456448.0, - "61": 2684456448.0, - "62": 2684456448.0, - "63": 2684456448.0, - "64": 2684456448.0, - "65": 2684456448.0, - "66": 2684456448.0, - "67": 2684456448.0, - "68": 2684456448.0, - "69": 2684456448.0, - "70": 2684456448.0, - "71": 2684456448.0, - "72": 2684456448.0, - "73": 2684456448.0, - "74": 2684456448.0, - "75": 2684456448.0, - "76": 2684456448.0, - "77": 2684456448.0, - "78": 2684456448.0, - "79": 2684456448.0, - "80": 2684456448.0, - "81": 2684456448.0, - "82": 2684456448.0, - "83": 2684456448.0, - "84": 2684456448.0, - "85": 2684456448.0, - "86": 2684456448.0, - "87": 2684456448.0, - "88": 2684456448.0, - "89": 2684456448.0, - "90": 2684456448.0, - "91": 2684456448.0, - "92": 2684456448.0, - "93": 2684456448.0, - "94": 2684456448.0, - "95": 2684456448.0, - "96": 2684456448.0, - "97": 2684456448.0, - "98": 2684456448.0, - "99": 2684456448.0, - "100": 2684456448.0 + "2": 2683145216.0, + "3": 2683145728.0, + "4": 2683145728.0, + "5": 2683145728.0, + "6": 2683145728.0, + "7": 2683145728.0, + "8": 2683145728.0, + "9": 2683147264.0, + "10": 2683147264.0, + "11": 2683147264.0, + "12": 2683147264.0, + "13": 2683147264.0, + "14": 2683147264.0, + "15": 2683147264.0, + "16": 2683147264.0, + "17": 2683147264.0, + "18": 2683670016.0, + "19": 2683670016.0, + "20": 2683670016.0, + "21": 2683670016.0, + "22": 2683670016.0, + "23": 2683670016.0, + "24": 2683670016.0, + "25": 2683670016.0, + "26": 2683671552.0, + "27": 2683671552.0, + "28": 2683671552.0, + "29": 2683671552.0, + "30": 2683671552.0, + "31": 2683671552.0, + "32": 2683671552.0, + "33": 2683671552.0, + "34": 2683671552.0, + "35": 2683671552.0, + "36": 2683671552.0, + "37": 2683671552.0, + "38": 2683671552.0, + "39": 2683671552.0, + "40": 2683671552.0, + "41": 2683671552.0, + "42": 2683671552.0, + "43": 2683671552.0, + "44": 2683671552.0, + "45": 2683671552.0, + "46": 2683671552.0, + "47": 2683671552.0, + "48": 2683671552.0, + "49": 2683671552.0, + "50": 2683671552.0, + "51": 2683671552.0, + "52": 2683671552.0, + "53": 2683671552.0, + "54": 2683671552.0, + "55": 2683671552.0, + "56": 2683671552.0, + "57": 2683671552.0, + "58": 2684195840.0, + "59": 2684195840.0, + "60": 2684195840.0, + "61": 2684195840.0, + "62": 2684195840.0, + "63": 2684195840.0, + "64": 2684195840.0, + "65": 2684195840.0, + "66": 2684195840.0, + "67": 2684195840.0, + "68": 2684195840.0, + "69": 2684195840.0, + "70": 2684195840.0, + "71": 2684195840.0, + "72": 2684195840.0, + "73": 2684195840.0, + "74": 2684195840.0, + "75": 2684195840.0, + "76": 2684195840.0, + "77": 2684195840.0, + "78": 2684195840.0, + "79": 2684195840.0, + "80": 2684195840.0, + "81": 2684195840.0, + "82": 2684195840.0, + "83": 2684195840.0, + "84": 2684195840.0, + "85": 2684195840.0, + "86": 2684195840.0, + "87": 2684195840.0, + "88": 2684195840.0, + "89": 2684195840.0, + "90": 2684195840.0, + "91": 2684195840.0, + "92": 2684195840.0, + "93": 2684195840.0, + "94": 2684195840.0, + "95": 2684195840.0, + "96": 2684195840.0, + "97": 2684195840.0, + "98": 2684195840.0, + "99": 2684195840.0, + "100": 2684195840.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.22036, - "3": 2.20415, - "4": 2.08777, - "5": 1.57617, - "6": 1.98982, - "7": 1.83939, - "8": 2.41823, - "9": 2.12984, - "10": 2.15958, - "11": 1.93646, - "12": 1.88825, - "13": 2.04226, - "14": 2.39522, - "15": 1.80906, - "16": 2.04254, - "17": 2.1028, - "18": 1.76417, - "19": 2.03236, - "20": 1.66314, - "21": 2.27582, - "22": 1.80755, - "23": 2.17162, - "24": 1.65212, - "25": 2.34768, - "26": 2.56374, - "27": 1.92359, - "28": 1.75954, - "29": 1.50854, - "30": 2.00136, - "31": 1.56852, - "32": 2.69666, - "33": 1.51766, - "34": 1.51194, - "35": 2.10968, - "36": 2.32577, - "37": 1.74564, - "38": 1.57205, - "39": 2.01796, - "40": 2.09859, - "41": 2.06025, - "42": 2.78705, - "43": 1.90579, - "44": 1.77323, - "45": 2.54723, - "46": 1.82766, - "47": 1.99145, - "48": 2.38637, - "49": 1.82776, - "50": 2.13381, - "51": 1.2664, - "52": 2.02135, - "53": 2.09316, - "54": 1.91073, - "55": 1.95887, - "56": 2.7439, - "57": 1.51213, - "58": 2.2268, - "59": 1.59326, - "60": 1.78325, - "61": 2.143, - "62": 2.24637, - "63": 1.77571, - "64": 2.08826, - "65": 1.88434, - "66": 2.2212, - "67": 2.18085, - "68": 1.68225, - "69": 2.26237, - "70": 1.89611, - "71": 2.41413, - "72": 1.8583, - "73": 1.85479, - "74": 1.7717, - "75": 1.83759, - "76": 1.98578, - "77": 1.86957, - "78": 1.7902, - "79": 2.30534, - "80": 1.99186, - "81": 2.46469, - "82": 2.251, - "83": 2.0939, - "84": 1.81168, - "85": 1.93932, - "86": 1.81157, - "87": 2.37052, - "88": 2.10101, - "89": 2.39261, - "90": 2.02708, - "91": 2.08531, - "92": 2.34193, - "93": 1.74529, - "94": 2.20042, - "95": 1.95062, - "96": 1.87083, - "97": 1.94129, - "98": 2.13306, - "99": 2.61932, - "100": 1.86229 + "2": 6.73204, + "3": 2.02558, + "4": 1.678, + "5": 1.35268, + "6": 1.45012, + "7": 1.38704, + "8": 1.91653, + "9": 1.88634, + "10": 1.6458, + "11": 1.77571, + "12": 1.554, + "13": 1.77562, + "14": 1.97892, + "15": 1.25673, + "16": 1.61015, + "17": 1.72685, + "18": 1.57976, + "19": 1.65927, + "20": 1.31968, + "21": 1.71494, + "22": 1.58058, + "23": 1.74262, + "24": 1.37335, + "25": 1.81061, + "26": 2.02255, + "27": 1.46079, + "28": 1.41093, + "29": 1.25729, + "30": 1.76274, + "31": 1.4288, + "32": 2.42726, + "33": 1.39911, + "34": 1.24607, + "35": 1.48244, + "36": 1.82737, + "37": 1.7932, + "38": 1.13855, + "39": 1.7021, + "40": 1.67538, + "41": 1.78056, + "42": 1.82611, + "43": 1.68084, + "44": 1.36721, + "45": 1.72367, + "46": 1.44735, + "47": 1.46296, + "48": 1.82044, + "49": 1.44258, + "50": 2.41821, + "51": 1.0391, + "52": 1.91439, + "53": 1.79236, + "54": 2.19063, + "55": 1.77536, + "56": 1.97945, + "57": 1.10557, + "58": 1.84107, + "59": 1.41976, + "60": 1.04037, + "61": 1.79345, + "62": 1.87028, + "63": 1.19971, + "64": 1.69364, + "65": 1.44315, + "66": 1.48665, + "67": 1.18753, + "68": 1.36045, + "69": 1.65243, + "70": 1.39734, + "71": 1.71946, + "72": 1.29397, + "73": 1.55519, + "74": 1.51156, + "75": 1.50307, + "76": 1.45086, + "77": 1.41402, + "78": 1.49313, + "79": 2.00303, + "80": 1.58581, + "81": 1.58386, + "82": 1.75209, + "83": 1.85996, + "84": 1.28763, + "85": 1.58727, + "86": 1.35617, + "87": 1.96274, + "88": 1.72849, + "89": 1.8081, + "90": 1.40748, + "91": 1.50952, + "92": 2.14963, + "93": 1.44106, + "94": 1.4261, + "95": 1.77967, + "96": 1.72259, + "97": 1.41216, + "98": 1.58913, + "99": 2.25859, + "100": 1.57024 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json index 84de2c54b87..9be206297f4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94183, - "2": 10.93668, - "3": 10.93421, - "4": 10.94284, - "5": 10.93215, - "6": 10.9333, - "7": 10.93826, - "8": 10.937, - "9": 10.933, - "10": 10.93587, - "11": 10.92319, - "12": 10.92246, - "13": 10.92256, - "14": 10.91552, - "15": 10.89065, - "16": 10.88925, - "17": 10.89994, - "18": 10.87885, - "19": 10.87357, + "1": 10.94173, + "2": 10.9367, + "3": 10.93429, + "4": 10.94283, + "5": 10.93212, + "6": 10.93333, + "7": 10.93833, + "8": 10.93699, + "9": 10.93307, + "10": 10.93589, + "11": 10.92316, + "12": 10.92241, + "13": 10.92261, + "14": 10.91548, + "15": 10.89074, + "16": 10.88934, + "17": 10.89991, + "18": 10.87881, + "19": 10.87365, "20": 10.79056, - "21": 10.78102, - "22": 10.77848, - "23": 10.77629, - "24": 10.73434, - "25": 10.74471, - "26": 10.72486, - "27": 10.69276, - "28": 10.62282, - "29": 10.60182, - "30": 10.58029, - "31": 10.56944, - "32": 10.54985, - "33": 10.52288, - "34": 10.48608, - "35": 10.49156, - "36": 10.47137, - "37": 10.43632, - "38": 10.4382, - "39": 10.40194, - "40": 10.39026, - "41": 10.37045, - "42": 10.34739, - "43": 10.32448, - "44": 10.29037, - "45": 10.30668, - "46": 10.26821, - "47": 10.25077, - "48": 10.20228, + "21": 10.78111, + "22": 10.77847, + "23": 10.77638, + "24": 10.73439, + "25": 10.74473, + "26": 10.72487, + "27": 10.69282, + "28": 10.62286, + "29": 10.60184, + "30": 10.58032, + "31": 10.56952, + "32": 10.54992, + "33": 10.52291, + "34": 10.48611, + "35": 10.49153, + "36": 10.4714, + "37": 10.43631, + "38": 10.43824, + "39": 10.40197, + "40": 10.39034, + "41": 10.37053, + "42": 10.34746, + "43": 10.32449, + "44": 10.29044, + "45": 10.30678, + "46": 10.26826, + "47": 10.25082, + "48": 10.20236, "49": 10.2013, - "50": 10.21031, - "51": 10.20451, - "52": 10.15574, - "53": 10.16784, - "54": 10.13048, - "55": 10.09999, - "56": 10.12944, - "57": 10.11355, - "58": 10.13078, - "59": 10.07364, - "60": 10.08692, - "61": 10.04213, - "62": 10.01099, - "63": 10.08369, - "64": 10.03748, - "65": 10.00903, - "66": 10.03921, - "67": 10.007, - "68": 9.97356, - "69": 9.99336, + "50": 10.21034, + "51": 10.20453, + "52": 10.15576, + "53": 10.16794, + "54": 10.13058, + "55": 10.10002, + "56": 10.12945, + "57": 10.11362, + "58": 10.13081, + "59": 10.07367, + "60": 10.08694, + "61": 10.04215, + "62": 10.011, + "63": 10.08374, + "64": 10.03755, + "65": 10.00906, + "66": 10.03922, + "67": 10.00698, + "68": 9.9736, + "69": 9.99337, "70": 9.98305, - "71": 10.0021, - "72": 9.97314, - "73": 9.96909, - "74": 9.95898, - "75": 9.92485, - "76": 9.96566, - "77": 9.95635, - "78": 9.90227, - "79": 9.91014, - "80": 9.92301, - "81": 9.94404, - "82": 9.88662, - "83": 9.84446, - "84": 9.78432, - "85": 9.77613, - "86": 9.87724, - "87": 9.91085, - "88": 9.88227, + "71": 10.00213, + "72": 9.9732, + "73": 9.96913, + "74": 9.95903, + "75": 9.92489, + "76": 9.96567, + "77": 9.95646, + "78": 9.90234, + "79": 9.91023, + "80": 9.92308, + "81": 9.94407, + "82": 9.88668, + "83": 9.84453, + "84": 9.78437, + "85": 9.77616, + "86": 9.87731, + "87": 9.91081, + "88": 9.88235, "89": 9.8201, - "90": 9.80697, - "91": 9.81928, - "92": 9.80885, - "93": 9.74747, - "94": 9.82086, - "95": 9.81606, - "96": 9.80232, - "97": 9.74008, - "98": 9.76949, + "90": 9.80704, + "91": 9.81933, + "92": 9.8089, + "93": 9.74751, + "94": 9.82092, + "95": 9.81607, + "96": 9.80238, + "97": 9.7401, + "98": 9.76956, "99": 9.81602, - "100": 9.70635 + "100": 9.70638 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1932.0, - "2": 1782.0, - "3": 1862.0, - "4": 1715.0, - "5": 1742.0, + "1": 1834.0, + "2": 1767.0, + "3": 1788.0, + "4": 1755.0, + "5": 1755.0, "6": 1720.0, - "7": 2040.0, - "8": 1780.0, - "9": 1895.0, - "10": 1743.0, - "11": 1801.0, - "12": 1654.0, - "13": 1813.0, - "14": 1784.0, - "15": 1751.0, - "16": 1748.0, - "17": 1827.0, - "18": 1839.0, - "19": 1659.0, - "20": 1742.0, - "21": 1825.0, - "22": 1677.0, - "23": 1715.0, - "24": 1858.0, - "25": 1812.0, - "26": 1838.0, - "27": 1789.0, - "28": 1839.0, - "29": 2001.0, - "30": 1832.0, - "31": 1962.0, - "32": 1990.0, - "33": 1996.0, - "34": 2077.0, - "35": 2065.0, - "36": 1951.0, - "37": 2323.0, - "38": 2067.0, - "39": 2125.0, - "40": 2280.0, - "41": 2363.0, - "42": 1962.0, - "43": 2226.0, - "44": 2175.0, - "45": 2534.0, - "46": 2445.0, - "47": 2416.0, - "48": 2574.0, - "49": 2865.0, - "50": 2588.0, - "51": 2442.0, - "52": 2682.0, - "53": 2492.0, - "54": 2756.0, - "55": 2486.0, - "56": 2637.0, - "57": 2216.0, - "58": 3466.0, - "59": 2849.0, - "60": 2889.0, - "61": 2661.0, - "62": 3068.0, - "63": 3164.0, - "64": 3417.0, - "65": 2612.0, - "66": 2955.0, - "67": 3844.0, - "68": 3541.0, - "69": 2904.0, - "70": 3184.0, - "71": 3108.0, - "72": 2962.0, - "73": 3189.0, - "74": 3163.0, - "75": 3197.0, - "76": 3136.0, - "77": 3702.0, - "78": 3046.0, - "79": 3258.0, - "80": 3094.0, - "81": 3444.0, - "82": 3141.0, - "83": 3094.0, - "84": 2998.0, - "85": 2761.0, - "86": 3081.0, - "87": 2960.0, - "88": 3011.0, - "89": 2786.0, - "90": 3562.0, - "91": 2966.0, - "92": 2835.0, - "93": 3211.0, - "94": 3145.0, - "95": 3403.0, - "96": 3519.0, - "97": 3574.0, - "98": 3569.0, - "99": 3279.0, - "100": 3277.0 + "7": 2043.0, + "8": 1813.0, + "9": 1774.0, + "10": 1831.0, + "11": 1724.0, + "12": 1671.0, + "13": 1758.0, + "14": 1804.0, + "15": 1721.0, + "16": 1738.0, + "17": 1831.0, + "18": 1900.0, + "19": 1782.0, + "20": 1565.0, + "21": 1862.0, + "22": 1775.0, + "23": 1691.0, + "24": 1922.0, + "25": 1884.0, + "26": 1781.0, + "27": 1847.0, + "28": 1847.0, + "29": 1940.0, + "30": 1852.0, + "31": 1950.0, + "32": 1978.0, + "33": 1899.0, + "34": 2013.0, + "35": 2023.0, + "36": 1906.0, + "37": 2202.0, + "38": 2089.0, + "39": 2071.0, + "40": 2311.0, + "41": 2273.0, + "42": 2010.0, + "43": 2386.0, + "44": 2220.0, + "45": 2564.0, + "46": 2420.0, + "47": 2480.0, + "48": 2562.0, + "49": 2758.0, + "50": 2614.0, + "51": 2400.0, + "52": 2680.0, + "53": 2524.0, + "54": 2819.0, + "55": 2509.0, + "56": 2660.0, + "57": 2202.0, + "58": 3533.0, + "59": 2812.0, + "60": 2952.0, + "61": 2716.0, + "62": 3000.0, + "63": 3156.0, + "64": 3564.0, + "65": 2660.0, + "66": 3078.0, + "67": 3748.0, + "68": 3343.0, + "69": 2872.0, + "70": 3100.0, + "71": 2960.0, + "72": 2978.0, + "73": 3318.0, + "74": 3145.0, + "75": 3094.0, + "76": 3108.0, + "77": 3803.0, + "78": 3150.0, + "79": 3192.0, + "80": 3064.0, + "81": 3467.0, + "82": 3160.0, + "83": 3144.0, + "84": 2910.0, + "85": 2684.0, + "86": 3097.0, + "87": 2855.0, + "88": 3107.0, + "89": 2934.0, + "90": 3533.0, + "91": 2976.0, + "92": 2928.0, + "93": 3048.0, + "94": 3220.0, + "95": 3400.0, + "96": 3348.0, + "97": 3730.0, + "98": 3584.0, + "99": 3327.0, + "100": 3288.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 551277056.0, - "2": 551277056.0, - "3": 551277056.0, - "4": 551277056.0, - "5": 551277056.0, - "6": 551277056.0, - "7": 551277056.0, - "8": 551277056.0, - "9": 551277056.0, - "10": 551277056.0, - "11": 551277056.0, - "12": 551277056.0, - "13": 551277056.0, - "14": 551277056.0, - "15": 551277056.0, - "16": 551277056.0, - "17": 551277056.0, - "18": 551277056.0, - "19": 551277056.0, - "20": 551277056.0, - "21": 551277056.0, - "22": 551277056.0, - "23": 551277056.0, - "24": 551277056.0, - "25": 551277056.0, - "26": 551277056.0, - "27": 551277056.0, - "28": 551277056.0, - "29": 551277056.0, - "30": 551277056.0, - "31": 551277056.0, - "32": 551277056.0, - "33": 551277056.0, - "34": 551277056.0, - "35": 551277056.0, - "36": 551277056.0, - "37": 551277056.0, - "38": 551277056.0, - "39": 551277056.0, - "40": 551277056.0, - "41": 551277056.0, - "42": 551277056.0, - "43": 551277056.0, - "44": 551277056.0, - "45": 551277056.0, - "46": 551277056.0, - "47": 551277056.0, - "48": 551277056.0, - "49": 551277056.0, - "50": 551277056.0, - "51": 551277056.0, - "52": 551277056.0, - "53": 551277056.0, - "54": 551277056.0, - "55": 551277056.0, - "56": 551277056.0, - "57": 551277056.0, - "58": 551277056.0, - "59": 551277056.0, - "60": 551277056.0, - "61": 551277056.0, - "62": 551277056.0, - "63": 551277056.0, - "64": 551277056.0, - "65": 551277056.0, - "66": 551277056.0, - "67": 551277056.0, - "68": 551277056.0, - "69": 551277056.0, - "70": 551277056.0, - "71": 551277056.0, - "72": 551277056.0, - "73": 551277056.0, - "74": 551277056.0, - "75": 551277056.0, - "76": 551277056.0, - "77": 551277056.0, - "78": 551277056.0, - "79": 551277056.0, - "80": 551277056.0, - "81": 551277056.0, - "82": 551277056.0, - "83": 551277056.0, - "84": 551277056.0, - "85": 551277056.0, - "86": 551277056.0, - "87": 551277056.0, - "88": 551277056.0, - "89": 551277056.0, - "90": 551277056.0, - "91": 551277056.0, - "92": 551277056.0, - "93": 551277056.0, - "94": 551277056.0, - "95": 551277056.0, - "96": 551277056.0, - "97": 551277056.0, - "98": 551277056.0, - "99": 551277056.0, - "100": 551277056.0 + "1": 551271936.0, + "2": 551271936.0, + "3": 551271936.0, + "4": 551271936.0, + "5": 551271936.0, + "6": 551271936.0, + "7": 551271936.0, + "8": 551271936.0, + "9": 551271936.0, + "10": 551271936.0, + "11": 551271936.0, + "12": 551271936.0, + "13": 551271936.0, + "14": 551271936.0, + "15": 551271936.0, + "16": 551271936.0, + "17": 551271936.0, + "18": 551271936.0, + "19": 551271936.0, + "20": 551271936.0, + "21": 551271936.0, + "22": 551271936.0, + "23": 551271936.0, + "24": 551271936.0, + "25": 551271936.0, + "26": 551271936.0, + "27": 551271936.0, + "28": 551271936.0, + "29": 551271936.0, + "30": 551271936.0, + "31": 551271936.0, + "32": 551271936.0, + "33": 551271936.0, + "34": 551271936.0, + "35": 551271936.0, + "36": 551271936.0, + "37": 551271936.0, + "38": 551271936.0, + "39": 551271936.0, + "40": 551271936.0, + "41": 551271936.0, + "42": 551271936.0, + "43": 551271936.0, + "44": 551271936.0, + "45": 551271936.0, + "46": 551271936.0, + "47": 551271936.0, + "48": 551271936.0, + "49": 551271936.0, + "50": 551271936.0, + "51": 551271936.0, + "52": 551271936.0, + "53": 551271936.0, + "54": 551271936.0, + "55": 551271936.0, + "56": 551271936.0, + "57": 551271936.0, + "58": 551271936.0, + "59": 551271936.0, + "60": 551271936.0, + "61": 551271936.0, + "62": 551271936.0, + "63": 551271936.0, + "64": 551271936.0, + "65": 551271936.0, + "66": 551271936.0, + "67": 551271936.0, + "68": 551271936.0, + "69": 551271936.0, + "70": 551271936.0, + "71": 551271936.0, + "72": 551271936.0, + "73": 551271936.0, + "74": 551271936.0, + "75": 551271936.0, + "76": 551271936.0, + "77": 551271936.0, + "78": 551271936.0, + "79": 551271936.0, + "80": 551271936.0, + "81": 551271936.0, + "82": 551271936.0, + "83": 551271936.0, + "84": 551271936.0, + "85": 551271936.0, + "86": 551271936.0, + "87": 551271936.0, + "88": 551271936.0, + "89": 551271936.0, + "90": 551271936.0, + "91": 551271936.0, + "92": 551271936.0, + "93": 551271936.0, + "94": 551271936.0, + "95": 551271936.0, + "96": 551271936.0, + "97": 551271936.0, + "98": 551271936.0, + "99": 551271936.0, + "100": 551271936.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2328238592.0, - "2": 2469104640.0, - "3": 2469104640.0, - "4": 2469104640.0, - "5": 2469104640.0, - "6": 2469104640.0, - "7": 2469104640.0, - "8": 2469104640.0, - "9": 2469104640.0, - "10": 2469104640.0, - "11": 2469104640.0, - "12": 2469104640.0, - "13": 2469104640.0, - "14": 2469104640.0, - "15": 2469104640.0, - "16": 2469104640.0, - "17": 2469104640.0, - "18": 2469104640.0, - "19": 2469104640.0, - "20": 2469104640.0, - "21": 2469104640.0, - "22": 2469104640.0, - "23": 2469104640.0, - "24": 2469104640.0, - "25": 2469104640.0, - "26": 2469104640.0, - "27": 2469104640.0, - "28": 2469104640.0, - "29": 2469104640.0, - "30": 2469104640.0, - "31": 2469104640.0, - "32": 2469104640.0, - "33": 2469104640.0, - "34": 2469104640.0, - "35": 2469104640.0, - "36": 2469104640.0, - "37": 2469104640.0, - "38": 2469104640.0, - "39": 2469104640.0, - "40": 2469104640.0, - "41": 2469104640.0, - "42": 2469104640.0, - "43": 2469104640.0, - "44": 2469104640.0, - "45": 2469104640.0, - "46": 2469104640.0, - "47": 2469104640.0, - "48": 2469104640.0, - "49": 2469104640.0, - "50": 2469104640.0, - "51": 2469104640.0, - "52": 2469104640.0, - "53": 2469104640.0, - "54": 2469104640.0, - "55": 2469104640.0, - "56": 2469104640.0, - "57": 2469104640.0, - "58": 2469104640.0, - "59": 2469104640.0, - "60": 2469104640.0, - "61": 2469104640.0, - "62": 2469104640.0, - "63": 2469104640.0, - "64": 2469104640.0, - "65": 2469104640.0, - "66": 2469104640.0, - "67": 2469104640.0, - "68": 2469104640.0, - "69": 2469104640.0, - "70": 2469104640.0, - "71": 2469104640.0, - "72": 2469104640.0, - "73": 2469104640.0, - "74": 2469104640.0, - "75": 2469104640.0, - "76": 2469104640.0, - "77": 2469104640.0, - "78": 2469104640.0, - "79": 2469104640.0, - "80": 2469104640.0, - "81": 2469104640.0, - "82": 2469104640.0, - "83": 2469104640.0, - "84": 2469104640.0, - "85": 2469104640.0, - "86": 2469104640.0, - "87": 2469104640.0, - "88": 2469104640.0, - "89": 2469104640.0, - "90": 2469104640.0, - "91": 2469104640.0, - "92": 2469104640.0, - "93": 2469104640.0, - "94": 2469104640.0, - "95": 2469104640.0, - "96": 2469104640.0, - "97": 2469104640.0, - "98": 2469104640.0, - "99": 2469104640.0, - "100": 2469104640.0 + "2": 2469099520.0, + "3": 2469099520.0, + "4": 2469099520.0, + "5": 2469099520.0, + "6": 2469099520.0, + "7": 2469099520.0, + "8": 2469099520.0, + "9": 2469099520.0, + "10": 2469099520.0, + "11": 2469099520.0, + "12": 2469099520.0, + "13": 2469099520.0, + "14": 2469099520.0, + "15": 2469099520.0, + "16": 2469099520.0, + "17": 2469099520.0, + "18": 2469099520.0, + "19": 2469099520.0, + "20": 2469099520.0, + "21": 2469099520.0, + "22": 2469099520.0, + "23": 2469099520.0, + "24": 2469099520.0, + "25": 2469099520.0, + "26": 2469099520.0, + "27": 2469099520.0, + "28": 2469099520.0, + "29": 2469099520.0, + "30": 2469099520.0, + "31": 2469099520.0, + "32": 2469099520.0, + "33": 2469099520.0, + "34": 2469099520.0, + "35": 2469099520.0, + "36": 2469099520.0, + "37": 2469099520.0, + "38": 2469099520.0, + "39": 2469099520.0, + "40": 2469099520.0, + "41": 2469099520.0, + "42": 2469099520.0, + "43": 2469099520.0, + "44": 2469099520.0, + "45": 2469099520.0, + "46": 2469099520.0, + "47": 2469099520.0, + "48": 2469099520.0, + "49": 2469099520.0, + "50": 2469099520.0, + "51": 2469099520.0, + "52": 2469099520.0, + "53": 2469099520.0, + "54": 2469099520.0, + "55": 2469099520.0, + "56": 2469099520.0, + "57": 2469099520.0, + "58": 2469099520.0, + "59": 2469099520.0, + "60": 2469099520.0, + "61": 2469099520.0, + "62": 2469099520.0, + "63": 2469099520.0, + "64": 2469099520.0, + "65": 2469099520.0, + "66": 2469099520.0, + "67": 2469099520.0, + "68": 2469099520.0, + "69": 2469099520.0, + "70": 2469099520.0, + "71": 2469099520.0, + "72": 2469099520.0, + "73": 2469099520.0, + "74": 2469099520.0, + "75": 2469099520.0, + "76": 2469099520.0, + "77": 2469099520.0, + "78": 2469099520.0, + "79": 2469099520.0, + "80": 2469099520.0, + "81": 2469099520.0, + "82": 2469099520.0, + "83": 2469099520.0, + "84": 2469099520.0, + "85": 2469099520.0, + "86": 2469099520.0, + "87": 2469099520.0, + "88": 2469099520.0, + "89": 2469099520.0, + "90": 2469099520.0, + "91": 2469099520.0, + "92": 2469099520.0, + "93": 2469099520.0, + "94": 2469099520.0, + "95": 2469099520.0, + "96": 2469099520.0, + "97": 2469099520.0, + "98": 2469099520.0, + "99": 2469099520.0, + "100": 2469099520.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.82345, - "3": 0.13014, - "4": 0.11363, - "5": 0.11445, - "6": 0.11134, - "7": 0.11139, - "8": 0.1134, - "9": 0.1143, - "10": 0.1121, - "11": 0.11262, - "12": 0.1124, - "13": 0.11413, - "14": 0.1128, - "15": 0.1127, - "16": 0.11088, - "17": 0.11314, - "18": 0.11264, - "19": 0.10972, - "20": 0.11329, - "21": 0.111, - "22": 0.11375, - "23": 0.11349, - "24": 0.1104, - "25": 0.11287, - "26": 0.1134, - "27": 0.11354, - "28": 0.11199, - "29": 0.11194, - "30": 0.11308, - "31": 0.11183, - "32": 0.11282, - "33": 0.11203, - "34": 0.11222, - "35": 0.1131, - "36": 0.11092, - "37": 0.11044, - "38": 0.11308, - "39": 0.11191, - "40": 0.1125, - "41": 0.11134, - "42": 0.11107, - "43": 0.10981, - "44": 0.11295, - "45": 0.11041, - "46": 0.11164, - "47": 0.11414, - "48": 0.11096, - "49": 0.11135, - "50": 0.11377, - "51": 0.27563, - "52": 0.13238, - "53": 0.12584, - "54": 0.14924, - "55": 0.11784, - "56": 0.1139, - "57": 0.11013, - "58": 0.10884, - "59": 0.11042, - "60": 0.11154, - "61": 0.11259, - "62": 0.11281, - "63": 0.1125, - "64": 0.11146, - "65": 0.11166, - "66": 0.11151, - "67": 0.11131, - "68": 0.11182, - "69": 0.11189, - "70": 0.11191, - "71": 0.11258, - "72": 0.11267, - "73": 0.11235, - "74": 0.1107, - "75": 0.11099, - "76": 0.10891, - "77": 0.11149, - "78": 0.11202, - "79": 0.11237, - "80": 0.11221, - "81": 0.11282, - "82": 0.11318, - "83": 0.11406, - "84": 0.11256, - "85": 0.11216, - "86": 0.11259, - "87": 0.11184, - "88": 0.11282, - "89": 0.11338, - "90": 0.11251, - "91": 0.11383, - "92": 0.11345, - "93": 0.11062, - "94": 0.11218, - "95": 0.11299, - "96": 0.11245, - "97": 0.1141, - "98": 0.11244, - "99": 0.11254, - "100": 0.11164 + "2": 11.14289, + "3": 0.15509, + "4": 0.10531, + "5": 0.10396, + "6": 0.10612, + "7": 0.10584, + "8": 0.10588, + "9": 0.10572, + "10": 0.10578, + "11": 0.10368, + "12": 0.10058, + "13": 0.10187, + "14": 0.10092, + "15": 0.10247, + "16": 0.10239, + "17": 0.10341, + "18": 0.10203, + "19": 0.10295, + "20": 0.10147, + "21": 0.10096, + "22": 0.10259, + "23": 0.10069, + "24": 0.10148, + "25": 0.1, + "26": 0.10226, + "27": 0.10078, + "28": 0.10028, + "29": 0.10085, + "30": 0.10011, + "31": 0.09922, + "32": 0.10048, + "33": 0.10031, + "34": 0.10037, + "35": 0.10003, + "36": 0.1019, + "37": 0.10051, + "38": 0.10211, + "39": 0.10114, + "40": 0.1057, + "41": 0.1038, + "42": 0.10085, + "43": 0.10231, + "44": 0.10336, + "45": 0.10089, + "46": 0.1025, + "47": 0.102, + "48": 0.10205, + "49": 0.09994, + "50": 0.10025, + "51": 0.27756, + "52": 0.15779, + "53": 0.10574, + "54": 0.10142, + "55": 0.0991, + "56": 0.11083, + "57": 0.09745, + "58": 0.09861, + "59": 0.10189, + "60": 0.09787, + "61": 0.09999, + "62": 0.09972, + "63": 0.09962, + "64": 0.0987, + "65": 0.0982, + "66": 0.09794, + "67": 0.11809, + "68": 0.09922, + "69": 0.09851, + "70": 0.09977, + "71": 0.09852, + "72": 0.10219, + "73": 0.10337, + "74": 0.10023, + "75": 0.10105, + "76": 0.10331, + "77": 0.10231, + "78": 0.10045, + "79": 0.09953, + "80": 0.09953, + "81": 0.10171, + "82": 0.22737, + "83": 0.10248, + "84": 0.10077, + "85": 0.10329, + "86": 0.09976, + "87": 0.09822, + "88": 0.10181, + "89": 0.10266, + "90": 0.09801, + "91": 0.10426, + "92": 0.2549, + "93": 0.10319, + "94": 0.09938, + "95": 0.10272, + "96": 0.10015, + "97": 0.24789, + "98": 0.10074, + "99": 0.10101, + "100": 0.10027 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json index a1335d718ac..2cf7a0b638b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95854, - "4": 10.95197, - "5": 10.95261, + "1": 10.96462, + "2": 10.95232, + "3": 10.95859, + "4": 10.95192, + "5": 10.95262, "6": 10.95382, - "7": 10.95159, - "8": 10.95132, - "9": 10.9587, - "10": 10.94456, - "11": 10.9368, - "12": 10.94339, + "7": 10.95158, + "8": 10.95135, + "9": 10.95869, + "10": 10.94452, + "11": 10.93685, + "12": 10.94335, "13": 10.94312, - "14": 10.94291, - "15": 10.91494, - "16": 10.90032, - "17": 10.91109, - "18": 10.90619, - "19": 10.90162, - "20": 10.80791, - "21": 10.7946, - "22": 10.80473, - "23": 10.78976, - "24": 10.77499, + "14": 10.94292, + "15": 10.9149, + "16": 10.90029, + "17": 10.91107, + "18": 10.90616, + "19": 10.90165, + "20": 10.80787, + "21": 10.79457, + "22": 10.80477, + "23": 10.78977, + "24": 10.77504, "25": 10.76169, - "26": 10.7552, - "27": 10.71953, - "28": 10.63914, - "29": 10.60852, - "30": 10.5865, - "31": 10.59099, - "32": 10.57106, - "33": 10.54032, - "34": 10.49898, - "35": 10.5003, - "36": 10.48632, - "37": 10.44935, - "38": 10.45072, - "39": 10.41769, - "40": 10.39539, - "41": 10.37479, - "42": 10.36062, - "43": 10.33044, - "44": 10.31902, - "45": 10.31601, - "46": 10.2765, - "47": 10.26822, - "48": 10.21836, - "49": 10.21385, - "50": 10.22195, - "51": 10.22215, - "52": 10.16751, + "26": 10.75515, + "27": 10.71954, + "28": 10.63906, + "29": 10.60855, + "30": 10.58656, + "31": 10.59096, + "32": 10.57113, + "33": 10.54034, + "34": 10.49902, + "35": 10.50031, + "36": 10.48629, + "37": 10.44938, + "38": 10.45069, + "39": 10.41772, + "40": 10.39543, + "41": 10.37489, + "42": 10.3607, + "43": 10.33047, + "44": 10.31907, + "45": 10.31615, + "46": 10.27658, + "47": 10.2683, + "48": 10.21832, + "49": 10.21392, + "50": 10.22203, + "51": 10.22219, + "52": 10.16759, "53": 10.17225, - "54": 10.13799, - "55": 10.10897, - "56": 10.13221, - "57": 10.12138, - "58": 10.12632, - "59": 10.07038, + "54": 10.138, + "55": 10.10895, + "56": 10.13222, + "57": 10.12137, + "58": 10.1263, + "59": 10.07041, "60": 10.09308, - "61": 10.04909, - "62": 10.01779, - "63": 10.08388, - "64": 10.03344, - "65": 10.00558, - "66": 10.04741, - "67": 10.02, - "68": 9.987, - "69": 9.99635, - "70": 9.98279, - "71": 10.00545, - "72": 9.99248, - "73": 9.97613, - "74": 9.96754, - "75": 9.93706, - "76": 9.96639, - "77": 9.96326, - "78": 9.91485, - "79": 9.91543, - "80": 9.92964, - "81": 9.95581, - "82": 9.89449, - "83": 9.85964, - "84": 9.80305, - "85": 9.7843, - "86": 9.88635, - "87": 9.90911, - "88": 9.88155, - "89": 9.82172, - "90": 9.82085, - "91": 9.82113, - "92": 9.81672, - "93": 9.74868, - "94": 9.82361, - "95": 9.81096, - "96": 9.79715, - "97": 9.74557, - "98": 9.76562, + "61": 10.0491, + "62": 10.01776, + "63": 10.08398, + "64": 10.03346, + "65": 10.0056, + "66": 10.04742, + "67": 10.02004, + "68": 9.98703, + "69": 9.99636, + "70": 9.98284, + "71": 10.0055, + "72": 9.99252, + "73": 9.9762, + "74": 9.9676, + "75": 9.93711, + "76": 9.96641, + "77": 9.96329, + "78": 9.91488, + "79": 9.9155, + "80": 9.92971, + "81": 9.95593, + "82": 9.89454, + "83": 9.85966, + "84": 9.80309, + "85": 9.78436, + "86": 9.88639, + "87": 9.90914, + "88": 9.88161, + "89": 9.82179, + "90": 9.82092, + "91": 9.82117, + "92": 9.81674, + "93": 9.7487, + "94": 9.82366, + "95": 9.81101, + "96": 9.7972, + "97": 9.74566, + "98": 9.76567, "99": 9.82205, - "100": 9.7044 + "100": 9.70445 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1787.0, - "4": 1675.0, - "5": 1820.0, - "6": 1678.0, - "7": 1861.0, - "8": 1749.0, - "9": 1745.0, - "10": 1830.0, - "11": 1702.0, - "12": 1714.0, - "13": 1843.0, - "14": 1903.0, - "15": 1605.0, - "16": 1760.0, - "17": 1770.0, - "18": 1856.0, - "19": 1689.0, - "20": 1673.0, - "21": 1780.0, - "22": 1786.0, - "23": 1719.0, - "24": 1842.0, + "1": 1759.0, + "2": 1644.0, + "3": 1736.0, + "4": 1783.0, + "5": 1782.0, + "6": 1759.0, + "7": 1975.0, + "8": 1703.0, + "9": 1724.0, + "10": 1760.0, + "11": 1678.0, + "12": 1698.0, + "13": 1776.0, + "14": 1809.0, + "15": 1596.0, + "16": 1706.0, + "17": 1765.0, + "18": 1843.0, + "19": 1667.0, + "20": 1696.0, + "21": 1746.0, + "22": 1714.0, + "23": 1724.0, + "24": 1912.0, "25": 1681.0, - "26": 1756.0, - "27": 1825.0, - "28": 1791.0, - "29": 1899.0, - "30": 1804.0, - "31": 2003.0, - "32": 1912.0, - "33": 1909.0, - "34": 1958.0, - "35": 2028.0, - "36": 1995.0, - "37": 2139.0, - "38": 2014.0, - "39": 2140.0, - "40": 2249.0, - "41": 2296.0, - "42": 2037.0, - "43": 2374.0, - "44": 2154.0, - "45": 2438.0, - "46": 2301.0, - "47": 2380.0, - "48": 2587.0, - "49": 2886.0, - "50": 2506.0, - "51": 2500.0, - "52": 2612.0, - "53": 2596.0, - "54": 2645.0, - "55": 2384.0, - "56": 2591.0, - "57": 2228.0, - "58": 3517.0, - "59": 2897.0, - "60": 2820.0, - "61": 2689.0, - "62": 3146.0, - "63": 3286.0, - "64": 3553.0, - "65": 2726.0, - "66": 3019.0, - "67": 3723.0, - "68": 3358.0, - "69": 2938.0, - "70": 3348.0, - "71": 3156.0, - "72": 2903.0, - "73": 3341.0, - "74": 3310.0, - "75": 3004.0, - "76": 3243.0, - "77": 3670.0, - "78": 3232.0, - "79": 3347.0, - "80": 3054.0, - "81": 3333.0, - "82": 2884.0, - "83": 3068.0, - "84": 2960.0, - "85": 2697.0, - "86": 3154.0, - "87": 2779.0, - "88": 3104.0, - "89": 3265.0, - "90": 3658.0, - "91": 2837.0, - "92": 3002.0, - "93": 3244.0, - "94": 2880.0, - "95": 3420.0, - "96": 3321.0, - "97": 3418.0, - "98": 3433.0, - "99": 3112.0, - "100": 2976.0 + "26": 1767.0, + "27": 1717.0, + "28": 1747.0, + "29": 1921.0, + "30": 1819.0, + "31": 1989.0, + "32": 1974.0, + "33": 1941.0, + "34": 1983.0, + "35": 2002.0, + "36": 1960.0, + "37": 2165.0, + "38": 2149.0, + "39": 2143.0, + "40": 2211.0, + "41": 2279.0, + "42": 2030.0, + "43": 2382.0, + "44": 2162.0, + "45": 2505.0, + "46": 2341.0, + "47": 2333.0, + "48": 2479.0, + "49": 2706.0, + "50": 2461.0, + "51": 2429.0, + "52": 2701.0, + "53": 2565.0, + "54": 2688.0, + "55": 2484.0, + "56": 2632.0, + "57": 2193.0, + "58": 3499.0, + "59": 2862.0, + "60": 3006.0, + "61": 2688.0, + "62": 3002.0, + "63": 3164.0, + "64": 3662.0, + "65": 2676.0, + "66": 3024.0, + "67": 3772.0, + "68": 3319.0, + "69": 3042.0, + "70": 3390.0, + "71": 3166.0, + "72": 2995.0, + "73": 3439.0, + "74": 3392.0, + "75": 3090.0, + "76": 3165.0, + "77": 3627.0, + "78": 3196.0, + "79": 3207.0, + "80": 3070.0, + "81": 3444.0, + "82": 2883.0, + "83": 3009.0, + "84": 2976.0, + "85": 2720.0, + "86": 3135.0, + "87": 2819.0, + "88": 3175.0, + "89": 3208.0, + "90": 3821.0, + "91": 2897.0, + "92": 2937.0, + "93": 3131.0, + "94": 2913.0, + "95": 3323.0, + "96": 3429.0, + "97": 3423.0, + "98": 3370.0, + "99": 3018.0, + "100": 3064.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 551148032.0, - "2": 551148032.0, - "3": 551148032.0, - "4": 551148032.0, - "5": 551148032.0, - "6": 551148032.0, - "7": 551148032.0, - "8": 551148032.0, - "9": 551148032.0, - "10": 551148032.0, - "11": 551148032.0, - "12": 551148032.0, - "13": 551148032.0, - "14": 551148032.0, - "15": 551148032.0, - "16": 551148032.0, - "17": 551148032.0, - "18": 551148032.0, - "19": 551148032.0, - "20": 551148032.0, - "21": 551148032.0, - "22": 551148032.0, - "23": 551148032.0, - "24": 551148032.0, - "25": 551148032.0, - "26": 551148032.0, - "27": 551148032.0, - "28": 551148032.0, - "29": 551148032.0, - "30": 551148032.0, - "31": 551148032.0, - "32": 551148032.0, - "33": 551148032.0, - "34": 551148032.0, - "35": 551148032.0, - "36": 551148032.0, - "37": 551148032.0, - "38": 551148032.0, - "39": 551148032.0, - "40": 551148032.0, - "41": 551148032.0, - "42": 551148032.0, - "43": 551148032.0, - "44": 551148032.0, - "45": 551148032.0, - "46": 551148032.0, - "47": 551148032.0, - "48": 551148032.0, - "49": 551148032.0, - "50": 551148032.0, - "51": 551148032.0, - "52": 551148032.0, - "53": 551148032.0, - "54": 551148032.0, - "55": 552065536.0, - "56": 551148032.0, - "57": 551148032.0, - "58": 551148032.0, - "59": 551148032.0, - "60": 551148032.0, - "61": 551148032.0, - "62": 551148032.0, - "63": 551148032.0, - "64": 551148032.0, - "65": 551148032.0, - "66": 551148032.0, - "67": 551148032.0, - "68": 551148032.0, - "69": 551148032.0, - "70": 551148032.0, - "71": 551148032.0, - "72": 551148032.0, - "73": 551148032.0, - "74": 551148032.0, - "75": 551148032.0, - "76": 551148032.0, - "77": 551148032.0, - "78": 551148032.0, - "79": 551148032.0, - "80": 551148032.0, - "81": 551148032.0, - "82": 551148032.0, - "83": 551148032.0, - "84": 551148032.0, - "85": 551148032.0, - "86": 551148032.0, - "87": 551148032.0, - "88": 552065536.0, - "89": 552065536.0, - "90": 551148032.0, - "91": 551148032.0, - "92": 551148032.0, - "93": 551148032.0, - "94": 551148032.0, - "95": 551148032.0, - "96": 551148032.0, - "97": 551148032.0, - "98": 551148032.0, - "99": 551148032.0, - "100": 551148032.0 + "1": 551137792.0, + "2": 551137792.0, + "3": 551137792.0, + "4": 551137792.0, + "5": 551137792.0, + "6": 551137792.0, + "7": 551137792.0, + "8": 551137792.0, + "9": 551137792.0, + "10": 551137792.0, + "11": 551137792.0, + "12": 551137792.0, + "13": 551137792.0, + "14": 551137792.0, + "15": 551137792.0, + "16": 551137792.0, + "17": 551137792.0, + "18": 551137792.0, + "19": 551137792.0, + "20": 551137792.0, + "21": 551137792.0, + "22": 551137792.0, + "23": 551137792.0, + "24": 551137792.0, + "25": 551137792.0, + "26": 551137792.0, + "27": 551137792.0, + "28": 551137792.0, + "29": 551137792.0, + "30": 551137792.0, + "31": 551137792.0, + "32": 551137792.0, + "33": 551137792.0, + "34": 551137792.0, + "35": 551137792.0, + "36": 551137792.0, + "37": 551137792.0, + "38": 551137792.0, + "39": 551137792.0, + "40": 551137792.0, + "41": 551137792.0, + "42": 551137792.0, + "43": 551137792.0, + "44": 551137792.0, + "45": 551137792.0, + "46": 551137792.0, + "47": 551137792.0, + "48": 551137792.0, + "49": 551137792.0, + "50": 551137792.0, + "51": 551137792.0, + "52": 551137792.0, + "53": 551137792.0, + "54": 551137792.0, + "55": 551137792.0, + "56": 551137792.0, + "57": 551137792.0, + "58": 551137792.0, + "59": 551137792.0, + "60": 552055296.0, + "61": 551137792.0, + "62": 551137792.0, + "63": 551137792.0, + "64": 552055296.0, + "65": 551137792.0, + "66": 551137792.0, + "67": 551137792.0, + "68": 551137792.0, + "69": 551137792.0, + "70": 551137792.0, + "71": 551137792.0, + "72": 551137792.0, + "73": 551137792.0, + "74": 551137792.0, + "75": 551137792.0, + "76": 551137792.0, + "77": 551137792.0, + "78": 551137792.0, + "79": 551137792.0, + "80": 551137792.0, + "81": 551137792.0, + "82": 551137792.0, + "83": 551137792.0, + "84": 551137792.0, + "85": 551137792.0, + "86": 551137792.0, + "87": 551137792.0, + "88": 551137792.0, + "89": 551137792.0, + "90": 551137792.0, + "91": 551137792.0, + "92": 551137792.0, + "93": 552055296.0, + "94": 551137792.0, + "95": 552055296.0, + "96": 551137792.0, + "97": 551137792.0, + "98": 551137792.0, + "99": 552055296.0, + "100": 551137792.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2290489856.0, - "2": 2431226880.0, - "3": 2431226880.0, - "4": 2431226880.0, - "5": 2431226880.0, - "6": 2431226880.0, - "7": 2431226880.0, - "8": 2431226880.0, - "9": 2431226880.0, - "10": 2431226880.0, - "11": 2431226880.0, - "12": 2431226880.0, - "13": 2431226880.0, - "14": 2431226880.0, - "15": 2431226880.0, - "16": 2431226880.0, - "17": 2431226880.0, - "18": 2431226880.0, - "19": 2431226880.0, - "20": 2431226880.0, - "21": 2431226880.0, - "22": 2431226880.0, - "23": 2431226880.0, - "24": 2431226880.0, - "25": 2431226880.0, - "26": 2431226880.0, - "27": 2431226880.0, - "28": 2431226880.0, - "29": 2431226880.0, - "30": 2431226880.0, - "31": 2431226880.0, - "32": 2431226880.0, - "33": 2431226880.0, - "34": 2431226880.0, - "35": 2431226880.0, - "36": 2431226880.0, - "37": 2431226880.0, - "38": 2431226880.0, - "39": 2431226880.0, - "40": 2431226880.0, - "41": 2431226880.0, - "42": 2431226880.0, - "43": 2431226880.0, - "44": 2431226880.0, - "45": 2431226880.0, - "46": 2431226880.0, - "47": 2431226880.0, - "48": 2431226880.0, - "49": 2431226880.0, - "50": 2431226880.0, - "51": 2432144384.0, - "52": 2432144384.0, - "53": 2432144384.0, - "54": 2432144384.0, - "55": 2432144384.0, - "56": 2432144384.0, - "57": 2432144384.0, - "58": 2432144384.0, - "59": 2432144384.0, - "60": 2432144384.0, - "61": 2432144384.0, - "62": 2432144384.0, - "63": 2432144384.0, - "64": 2432144384.0, - "65": 2432144384.0, - "66": 2432144384.0, - "67": 2432144384.0, - "68": 2432144384.0, - "69": 2432144384.0, - "70": 2432144384.0, - "71": 2432144384.0, - "72": 2432144384.0, - "73": 2432144384.0, - "74": 2432144384.0, - "75": 2432144384.0, - "76": 2432144384.0, - "77": 2432144384.0, - "78": 2432144384.0, - "79": 2432144384.0, - "80": 2432144384.0, - "81": 2432144384.0, - "82": 2432144384.0, - "83": 2432144384.0, - "84": 2432144384.0, - "85": 2432144384.0, - "86": 2432144384.0, - "87": 2432144384.0, - "88": 2432144384.0, - "89": 2432144384.0, - "90": 2432144384.0, - "91": 2432144384.0, - "92": 2432144384.0, - "93": 2432144384.0, - "94": 2432144384.0, - "95": 2432144384.0, - "96": 2432144384.0, - "97": 2432144384.0, - "98": 2432144384.0, - "99": 2432144384.0, - "100": 2432144384.0 + "2": 2431216640.0, + "3": 2431216640.0, + "4": 2431216640.0, + "5": 2431216640.0, + "6": 2431216640.0, + "7": 2431216640.0, + "8": 2431216640.0, + "9": 2431216640.0, + "10": 2431216640.0, + "11": 2431216640.0, + "12": 2431216640.0, + "13": 2431216640.0, + "14": 2431216640.0, + "15": 2431216640.0, + "16": 2431216640.0, + "17": 2431216640.0, + "18": 2431216640.0, + "19": 2431216640.0, + "20": 2431216640.0, + "21": 2431216640.0, + "22": 2431216640.0, + "23": 2431216640.0, + "24": 2431216640.0, + "25": 2431216640.0, + "26": 2431216640.0, + "27": 2431216640.0, + "28": 2431216640.0, + "29": 2431216640.0, + "30": 2431216640.0, + "31": 2431216640.0, + "32": 2431216640.0, + "33": 2431216640.0, + "34": 2431216640.0, + "35": 2431216640.0, + "36": 2431216640.0, + "37": 2431216640.0, + "38": 2431216640.0, + "39": 2431216640.0, + "40": 2431216640.0, + "41": 2431216640.0, + "42": 2431216640.0, + "43": 2431216640.0, + "44": 2431216640.0, + "45": 2431216640.0, + "46": 2431216640.0, + "47": 2431216640.0, + "48": 2431216640.0, + "49": 2431216640.0, + "50": 2431216640.0, + "51": 2432133632.0, + "52": 2432133632.0, + "53": 2432134144.0, + "54": 2432134144.0, + "55": 2432134144.0, + "56": 2432134144.0, + "57": 2432134144.0, + "58": 2432134144.0, + "59": 2432134144.0, + "60": 2432134144.0, + "61": 2432134144.0, + "62": 2432134144.0, + "63": 2432134144.0, + "64": 2432134144.0, + "65": 2432134144.0, + "66": 2432134144.0, + "67": 2432134144.0, + "68": 2432134144.0, + "69": 2432134144.0, + "70": 2432134144.0, + "71": 2432134144.0, + "72": 2432134144.0, + "73": 2432134144.0, + "74": 2432134144.0, + "75": 2432134144.0, + "76": 2432134144.0, + "77": 2432134144.0, + "78": 2432134144.0, + "79": 2432134144.0, + "80": 2432134144.0, + "81": 2432134144.0, + "82": 2432134144.0, + "83": 2432134144.0, + "84": 2432134144.0, + "85": 2432134144.0, + "86": 2432134144.0, + "87": 2432134144.0, + "88": 2432134144.0, + "89": 2432134144.0, + "90": 2432134144.0, + "91": 2432134144.0, + "92": 2432134144.0, + "93": 2432134144.0, + "94": 2432134144.0, + "95": 2432134144.0, + "96": 2432134144.0, + "97": 2432134144.0, + "98": 2432134144.0, + "99": 2432134144.0, + "100": 2432134144.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.13132, - "3": 0.11546, - "4": 0.11539, - "5": 0.11281, - "6": 0.11373, - "7": 0.11124, - "8": 0.11125, - "9": 0.11028, - "10": 0.11267, - "11": 0.11493, - "12": 0.11184, - "13": 0.11457, - "14": 0.11048, - "15": 0.10982, - "16": 0.1125, - "17": 0.11157, - "18": 0.11451, - "19": 0.11475, - "20": 0.11218, - "21": 0.11182, - "22": 0.11475, - "23": 0.11115, - "24": 0.11448, - "25": 0.11651, - "26": 0.11267, - "27": 0.11609, - "28": 0.11474, - "29": 0.11569, - "30": 0.11931, - "31": 0.11221, - "32": 0.11228, - "33": 0.11303, - "34": 0.11298, - "35": 0.11264, - "36": 0.11292, - "37": 0.11259, - "38": 0.11129, - "39": 0.11349, - "40": 0.11417, - "41": 0.11467, - "42": 0.11487, - "43": 0.11303, - "44": 0.1153, - "45": 0.11421, - "46": 0.11423, - "47": 0.11489, - "48": 0.11482, - "49": 0.11902, - "50": 0.11667, - "51": 0.24999, - "52": 0.14548, - "53": 0.11773, - "54": 0.11384, - "55": 0.1174, - "56": 0.11969, - "57": 0.1174, - "58": 0.1164, - "59": 0.11699, - "60": 0.11224, - "61": 0.11664, - "62": 0.1149, - "63": 0.11383, - "64": 0.11567, - "65": 0.11293, - "66": 0.11406, - "67": 0.11323, - "68": 0.11436, - "69": 0.11367, - "70": 0.11422, - "71": 0.11373, - "72": 0.12142, - "73": 0.12045, - "74": 0.12039, - "75": 0.12131, - "76": 0.11987, - "77": 0.11191, - "78": 0.11544, - "79": 0.11753, - "80": 0.11503, - "81": 0.11188, - "82": 0.11582, - "83": 0.11361, - "84": 0.11973, - "85": 0.12213, - "86": 0.11643, - "87": 0.12525, - "88": 0.12257, - "89": 0.11041, - "90": 0.11059, - "91": 0.10862, - "92": 0.11213, - "93": 0.11143, - "94": 0.11018, - "95": 0.11348, - "96": 0.11227, - "97": 0.11254, - "98": 0.11071, - "99": 0.10993, - "100": 0.11268 + "2": 9.66591, + "3": 5.85159, + "4": 3.47083, + "5": 2.53333, + "6": 3.20085, + "7": 3.73267, + "8": 4.06262, + "9": 4.45646, + "10": 2.41991, + "11": 3.83948, + "12": 3.38195, + "13": 3.92651, + "14": 2.67106, + "15": 2.61985, + "16": 3.28149, + "17": 3.14909, + "18": 3.22071, + "19": 3.48126, + "20": 4.17767, + "21": 3.64287, + "22": 3.10986, + "23": 3.64865, + "24": 4.56587, + "25": 1.94713, + "26": 2.50613, + "27": 3.00154, + "28": 2.92687, + "29": 1.87038, + "30": 1.93733, + "31": 3.03679, + "32": 3.43918, + "33": 4.37189, + "34": 1.39026, + "35": 3.82459, + "36": 3.02839, + "37": 3.0985, + "38": 1.64177, + "39": 2.67893, + "40": 3.6017, + "41": 2.37203, + "42": 3.13303, + "43": 2.10841, + "44": 3.26492, + "45": 2.0959, + "46": 2.4033, + "47": 2.92842, + "48": 2.86494, + "49": 2.5976, + "50": 3.28625, + "51": 1.17548, + "52": 3.17429, + "53": 3.28775, + "54": 2.75704, + "55": 3.28336, + "56": 3.45163, + "57": 2.26639, + "58": 3.8164, + "59": 2.7959, + "60": 1.65398, + "61": 3.26971, + "62": 2.27394, + "63": 2.26763, + "64": 3.2302, + "65": 2.43325, + "66": 2.72793, + "67": 3.05084, + "68": 2.07683, + "69": 2.5924, + "70": 3.65053, + "71": 3.41084, + "72": 1.56214, + "73": 3.01204, + "74": 1.99464, + "75": 2.32531, + "76": 2.97903, + "77": 2.5854, + "78": 3.37856, + "79": 3.89462, + "80": 2.88359, + "81": 1.96932, + "82": 3.5069, + "83": 3.05156, + "84": 2.35177, + "85": 1.48272, + "86": 2.05464, + "87": 4.45046, + "88": 1.21339, + "89": 2.54698, + "90": 2.11869, + "91": 2.1979, + "92": 3.122, + "93": 1.79965, + "94": 3.08931, + "95": 2.76042, + "96": 2.46304, + "97": 2.13059, + "98": 2.10301, + "99": 4.03004, + "100": 2.29975 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json index c860b494165..52d92ba336a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.9423, - "5": 10.93352, - "6": 10.93273, - "7": 10.93793, - "8": 10.93357, - "9": 10.93553, - "10": 10.93697, - "11": 10.92587, - "12": 10.92415, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.94226, + "5": 10.93351, + "6": 10.93269, + "7": 10.93786, + "8": 10.93354, + "9": 10.93555, + "10": 10.93689, + "11": 10.92583, + "12": 10.92411, "13": 10.92509, - "14": 10.9126, + "14": 10.91251, "15": 10.89336, - "16": 10.88824, - "17": 10.89973, - "18": 10.87823, - "19": 10.87587, - "20": 10.79123, - "21": 10.78019, - "22": 10.77476, - "23": 10.77499, + "16": 10.88832, + "17": 10.89971, + "18": 10.87815, + "19": 10.87589, + "20": 10.79125, + "21": 10.78018, + "22": 10.77478, + "23": 10.77507, "24": 10.73642, - "25": 10.74387, + "25": 10.74386, "26": 10.72466, "27": 10.6963, - "28": 10.62351, - "29": 10.6038, - "30": 10.58045, - "31": 10.56963, - "32": 10.54858, - "33": 10.52048, - "34": 10.48554, - "35": 10.49228, - "36": 10.4709, - "37": 10.43637, - "38": 10.44107, - "39": 10.40279, - "40": 10.39284, - "41": 10.36864, - "42": 10.34611, - "43": 10.32631, - "44": 10.28964, - "45": 10.30829, - "46": 10.26822, - "47": 10.25249, - "48": 10.2037, - "49": 10.20119, - "50": 10.20925, - "51": 10.2053, - "52": 10.15734, - "53": 10.16687, - "54": 10.13043, - "55": 10.10057, - "56": 10.13011, - "57": 10.11464, - "58": 10.13007, + "28": 10.62356, + "29": 10.60383, + "30": 10.58046, + "31": 10.56962, + "32": 10.54867, + "33": 10.52059, + "34": 10.48557, + "35": 10.4923, + "36": 10.47098, + "37": 10.43638, + "38": 10.44115, + "39": 10.40285, + "40": 10.39287, + "41": 10.36868, + "42": 10.34615, + "43": 10.3263, + "44": 10.28968, + "45": 10.30834, + "46": 10.26832, + "47": 10.25254, + "48": 10.20377, + "49": 10.20127, + "50": 10.20928, + "51": 10.20532, + "52": 10.15741, + "53": 10.16688, + "54": 10.13045, + "55": 10.10059, + "56": 10.13012, + "57": 10.11465, + "58": 10.13009, "59": 10.07596, - "60": 10.08697, - "61": 10.04223, - "62": 10.00953, - "63": 10.08368, - "64": 10.03719, - "65": 10.00997, - "66": 10.04022, - "67": 10.00689, - "68": 9.97342, - "69": 9.99439, - "70": 9.98223, - "71": 10.00185, - "72": 9.97365, - "73": 9.96699, - "74": 9.95725, - "75": 9.92477, + "60": 10.08704, + "61": 10.04224, + "62": 10.00959, + "63": 10.08375, + "64": 10.03718, + "65": 10.00996, + "66": 10.04027, + "67": 10.00691, + "68": 9.97351, + "69": 9.99451, + "70": 9.98228, + "71": 10.00188, + "72": 9.97372, + "73": 9.96702, + "74": 9.95733, + "75": 9.92481, "76": 9.96624, "77": 9.95575, - "78": 9.90173, - "79": 9.90898, - "80": 9.92106, - "81": 9.94319, - "82": 9.88581, - "83": 9.84494, - "84": 9.78312, - "85": 9.77444, - "86": 9.87683, - "87": 9.91005, - "88": 9.882, - "89": 9.81987, - "90": 9.80797, - "91": 9.81902, - "92": 9.80781, - "93": 9.74749, - "94": 9.82029, - "95": 9.8157, - "96": 9.80355, - "97": 9.7406, - "98": 9.76895, - "99": 9.8159, - "100": 9.70554 + "78": 9.90176, + "79": 9.90904, + "80": 9.9211, + "81": 9.94325, + "82": 9.88582, + "83": 9.84498, + "84": 9.78316, + "85": 9.77448, + "86": 9.87692, + "87": 9.91006, + "88": 9.88206, + "89": 9.81986, + "90": 9.80799, + "91": 9.81903, + "92": 9.80785, + "93": 9.74755, + "94": 9.82036, + "95": 9.81577, + "96": 9.80358, + "97": 9.74069, + "98": 9.76901, + "99": 9.81594, + "100": 9.70562 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1721.0, - "4": 1751.0, - "5": 1756.0, - "6": 1766.0, - "7": 2096.0, - "8": 1713.0, - "9": 1804.0, - "10": 1718.0, - "11": 1736.0, - "12": 1724.0, - "13": 1775.0, - "14": 1926.0, - "15": 1658.0, - "16": 1757.0, - "17": 1812.0, - "18": 1879.0, - "19": 1710.0, - "20": 1679.0, - "21": 1723.0, - "22": 1690.0, - "23": 1753.0, - "24": 1892.0, - "25": 1767.0, - "26": 1869.0, - "27": 1862.0, - "28": 1902.0, - "29": 1826.0, - "30": 1880.0, - "31": 1998.0, - "32": 1982.0, - "33": 1996.0, - "34": 2175.0, - "35": 2051.0, + "1": 1791.0, + "2": 1794.0, + "3": 1774.0, + "4": 1707.0, + "5": 1776.0, + "6": 1774.0, + "7": 2058.0, + "8": 1681.0, + "9": 1838.0, + "10": 1789.0, + "11": 1706.0, + "12": 1660.0, + "13": 1813.0, + "14": 1785.0, + "15": 1647.0, + "16": 1709.0, + "17": 1784.0, + "18": 1800.0, + "19": 1627.0, + "20": 1761.0, + "21": 1790.0, + "22": 1753.0, + "23": 1740.0, + "24": 1876.0, + "25": 1709.0, + "26": 1892.0, + "27": 1821.0, + "28": 1769.0, + "29": 1910.0, + "30": 1721.0, + "31": 2025.0, + "32": 1968.0, + "33": 2014.0, + "34": 1941.0, + "35": 2114.0, "36": 1974.0, - "37": 2256.0, - "38": 2159.0, - "39": 2063.0, - "40": 2239.0, - "41": 2379.0, - "42": 1944.0, - "43": 2319.0, - "44": 2172.0, - "45": 2550.0, - "46": 2381.0, - "47": 2453.0, - "48": 2591.0, - "49": 2636.0, - "50": 2590.0, - "51": 2477.0, - "52": 2793.0, - "53": 2561.0, - "54": 2717.0, - "55": 2472.0, - "56": 2677.0, - "57": 2183.0, - "58": 3495.0, - "59": 2935.0, - "60": 2895.0, - "61": 2757.0, - "62": 3041.0, - "63": 3137.0, - "64": 3332.0, - "65": 2609.0, - "66": 3019.0, - "67": 3698.0, - "68": 3370.0, - "69": 2933.0, - "70": 3278.0, - "71": 3092.0, - "72": 3047.0, - "73": 3310.0, - "74": 3031.0, - "75": 3167.0, - "76": 3175.0, - "77": 3706.0, - "78": 3237.0, - "79": 3195.0, - "80": 3060.0, - "81": 3281.0, - "82": 3156.0, - "83": 3213.0, - "84": 2976.0, - "85": 2832.0, - "86": 3080.0, - "87": 2995.0, - "88": 3282.0, - "89": 2781.0, - "90": 3476.0, - "91": 2998.0, - "92": 2978.0, - "93": 3222.0, - "94": 3236.0, - "95": 3461.0, - "96": 3503.0, - "97": 3660.0, - "98": 3618.0, - "99": 3139.0, - "100": 3325.0 + "37": 2192.0, + "38": 2098.0, + "39": 2200.0, + "40": 2326.0, + "41": 2322.0, + "42": 1973.0, + "43": 2366.0, + "44": 2124.0, + "45": 2480.0, + "46": 2376.0, + "47": 2474.0, + "48": 2571.0, + "49": 2729.0, + "50": 2520.0, + "51": 2535.0, + "52": 2643.0, + "53": 2500.0, + "54": 2804.0, + "55": 2465.0, + "56": 2642.0, + "57": 2279.0, + "58": 3437.0, + "59": 2900.0, + "60": 2882.0, + "61": 2663.0, + "62": 3087.0, + "63": 3166.0, + "64": 3453.0, + "65": 2565.0, + "66": 3047.0, + "67": 3793.0, + "68": 3427.0, + "69": 2948.0, + "70": 3106.0, + "71": 3002.0, + "72": 2919.0, + "73": 3334.0, + "74": 3171.0, + "75": 3022.0, + "76": 3089.0, + "77": 3742.0, + "78": 3125.0, + "79": 3174.0, + "80": 2973.0, + "81": 3351.0, + "82": 3108.0, + "83": 3161.0, + "84": 2969.0, + "85": 2740.0, + "86": 3109.0, + "87": 2940.0, + "88": 3162.0, + "89": 2814.0, + "90": 3576.0, + "91": 3124.0, + "92": 3001.0, + "93": 3157.0, + "94": 3170.0, + "95": 3476.0, + "96": 3481.0, + "97": 3689.0, + "98": 3612.0, + "99": 3224.0, + "100": 3437.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 761633280.0, - "2": 761633280.0, - "3": 761633280.0, - "4": 761633280.0, - "5": 761633280.0, - "6": 761633280.0, - "7": 761633280.0, - "8": 761633280.0, - "9": 761633280.0, - "10": 761633280.0, - "11": 761633280.0, - "12": 761633280.0, - "13": 761633280.0, - "14": 761633280.0, - "15": 761633280.0, - "16": 761633280.0, - "17": 761633280.0, - "18": 761633280.0, - "19": 761633280.0, - "20": 761633280.0, - "21": 761633280.0, - "22": 761633280.0, - "23": 761633280.0, - "24": 761633280.0, - "25": 761633280.0, - "26": 761633280.0, - "27": 761633280.0, - "28": 761633280.0, - "29": 761633280.0, - "30": 761633280.0, - "31": 761633280.0, - "32": 761633280.0, - "33": 761633280.0, - "34": 761633280.0, - "35": 761633280.0, - "36": 761633280.0, - "37": 761633280.0, - "38": 761633280.0, - "39": 761633280.0, - "40": 761633280.0, - "41": 761633280.0, - "42": 761633280.0, - "43": 761633280.0, - "44": 761633280.0, - "45": 761633280.0, - "46": 761633280.0, - "47": 761633280.0, - "48": 761633280.0, - "49": 761633280.0, - "50": 761633280.0, - "51": 761633280.0, - "52": 761633280.0, - "53": 761633280.0, - "54": 761633280.0, - "55": 761633280.0, - "56": 761633280.0, - "57": 761633280.0, - "58": 761633280.0, - "59": 761633280.0, - "60": 761633280.0, - "61": 761633280.0, - "62": 761633280.0, - "63": 761633280.0, - "64": 761633280.0, - "65": 761633280.0, - "66": 761633280.0, - "67": 761633280.0, - "68": 761633280.0, - "69": 761633280.0, - "70": 761633280.0, - "71": 761633280.0, - "72": 761633280.0, - "73": 761633280.0, - "74": 761633280.0, - "75": 761633280.0, - "76": 761633280.0, - "77": 761633280.0, - "78": 761633280.0, - "79": 761633280.0, - "80": 761633280.0, - "81": 761633280.0, - "82": 761633280.0, - "83": 761633280.0, - "84": 761633280.0, - "85": 761633280.0, - "86": 761633280.0, - "87": 761633280.0, - "88": 761633280.0, - "89": 761633280.0, - "90": 761633280.0, - "91": 761633280.0, - "92": 761633280.0, - "93": 761633280.0, - "94": 761633280.0, - "95": 761633280.0, - "96": 761633280.0, - "97": 761633280.0, - "98": 761633280.0, - "99": 761633280.0, - "100": 761633280.0 + "1": 760584704.0, + "2": 760584704.0, + "3": 760584704.0, + "4": 760584704.0, + "5": 760584704.0, + "6": 760584704.0, + "7": 760584704.0, + "8": 760584704.0, + "9": 760584704.0, + "10": 760584704.0, + "11": 760584704.0, + "12": 760584704.0, + "13": 760584704.0, + "14": 760584704.0, + "15": 760584704.0, + "16": 760584704.0, + "17": 760584704.0, + "18": 760584704.0, + "19": 760584704.0, + "20": 760584704.0, + "21": 760584704.0, + "22": 760584704.0, + "23": 760584704.0, + "24": 760584704.0, + "25": 760584704.0, + "26": 760584704.0, + "27": 760584704.0, + "28": 760584704.0, + "29": 760584704.0, + "30": 760584704.0, + "31": 760584704.0, + "32": 760584704.0, + "33": 760584704.0, + "34": 760584704.0, + "35": 760584704.0, + "36": 760584704.0, + "37": 760584704.0, + "38": 760584704.0, + "39": 760584704.0, + "40": 760584704.0, + "41": 760584704.0, + "42": 760584704.0, + "43": 760584704.0, + "44": 760584704.0, + "45": 760584704.0, + "46": 760584704.0, + "47": 760584704.0, + "48": 760584704.0, + "49": 760584704.0, + "50": 760584704.0, + "51": 760584704.0, + "52": 760584704.0, + "53": 760584704.0, + "54": 760584704.0, + "55": 760584704.0, + "56": 760584704.0, + "57": 760584704.0, + "58": 760584704.0, + "59": 760584704.0, + "60": 760584704.0, + "61": 760584704.0, + "62": 760584704.0, + "63": 760584704.0, + "64": 760584704.0, + "65": 760584704.0, + "66": 760584704.0, + "67": 760584704.0, + "68": 760584704.0, + "69": 760584704.0, + "70": 760584704.0, + "71": 760584704.0, + "72": 760584704.0, + "73": 760584704.0, + "74": 760584704.0, + "75": 760584704.0, + "76": 760584704.0, + "77": 760584704.0, + "78": 760584704.0, + "79": 760584704.0, + "80": 760584704.0, + "81": 760584704.0, + "82": 760584704.0, + "83": 760584704.0, + "84": 760584704.0, + "85": 760584704.0, + "86": 760584704.0, + "87": 760584704.0, + "88": 760584704.0, + "89": 760584704.0, + "90": 760584704.0, + "91": 760584704.0, + "92": 760584704.0, + "93": 760584704.0, + "94": 760584704.0, + "95": 760584704.0, + "96": 760584704.0, + "97": 760584704.0, + "98": 760584704.0, + "99": 760584704.0, + "100": 760584704.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2398797312.0, - "2": 2681558016.0, - "3": 2681560064.0, - "4": 2681560064.0, - "5": 2682084352.0, - "6": 2682084352.0, - "7": 2682084352.0, - "8": 2682084352.0, - "9": 2682084352.0, - "10": 2682084352.0, - "11": 2682084352.0, - "12": 2682084352.0, - "13": 2682084352.0, - "14": 2682084352.0, - "15": 2682084352.0, - "16": 2682084352.0, - "17": 2682084352.0, - "18": 2682084352.0, - "19": 2682084352.0, - "20": 2682084352.0, - "21": 2682084352.0, - "22": 2682084352.0, - "23": 2682084352.0, - "24": 2682084352.0, - "25": 2682084352.0, - "26": 2682084352.0, - "27": 2682084352.0, - "28": 2682084352.0, - "29": 2682084352.0, - "30": 2682084352.0, - "31": 2682084352.0, - "32": 2682084352.0, - "33": 2682084352.0, - "34": 2682084352.0, - "35": 2682084352.0, - "36": 2682084352.0, - "37": 2682084352.0, - "38": 2682084352.0, - "39": 2682084352.0, - "40": 2682084352.0, - "41": 2682084352.0, - "42": 2682084352.0, - "43": 2682084352.0, - "44": 2682084352.0, - "45": 2682084352.0, - "46": 2682084352.0, - "47": 2682084352.0, - "48": 2682084352.0, - "49": 2682084352.0, - "50": 2682084352.0, - "51": 2682084352.0, - "52": 2682084352.0, - "53": 2682084352.0, - "54": 2682084352.0, - "55": 2682084352.0, - "56": 2682084352.0, - "57": 2682084352.0, - "58": 2682084352.0, - "59": 2682084352.0, - "60": 2682084352.0, - "61": 2682084352.0, - "62": 2682084352.0, - "63": 2682084352.0, - "64": 2682084352.0, - "65": 2682084352.0, - "66": 2682084352.0, - "67": 2682084352.0, - "68": 2682084352.0, - "69": 2682084352.0, - "70": 2682084352.0, - "71": 2682084352.0, - "72": 2682084352.0, - "73": 2682084352.0, - "74": 2682084352.0, - "75": 2682084352.0, - "76": 2682084352.0, - "77": 2682084352.0, - "78": 2682084352.0, - "79": 2682084352.0, - "80": 2682607104.0, - "81": 2682607104.0, - "82": 2682607104.0, - "83": 2682607104.0, - "84": 2682607104.0, - "85": 2682607104.0, - "86": 2682607104.0, - "87": 2682607104.0, - "88": 2682607104.0, - "89": 2682607104.0, - "90": 2682607104.0, - "91": 2682607104.0, - "92": 2682607104.0, - "93": 2682607104.0, - "94": 2682607104.0, - "95": 2682607104.0, - "96": 2682607104.0, - "97": 2682607104.0, - "98": 2682607104.0, - "99": 2682607104.0, - "100": 2682607104.0 + "2": 2679460864.0, + "3": 2679462912.0, + "4": 2679462912.0, + "5": 2679462912.0, + "6": 2679462912.0, + "7": 2679462912.0, + "8": 2679987200.0, + "9": 2679987200.0, + "10": 2679987200.0, + "11": 2680509952.0, + "12": 2680509952.0, + "13": 2680509952.0, + "14": 2680509952.0, + "15": 2680509952.0, + "16": 2680509952.0, + "17": 2680509952.0, + "18": 2680509952.0, + "19": 2680509952.0, + "20": 2680509952.0, + "21": 2680509952.0, + "22": 2680509952.0, + "23": 2680509952.0, + "24": 2680509952.0, + "25": 2680509952.0, + "26": 2680509952.0, + "27": 2680509952.0, + "28": 2680509952.0, + "29": 2680509952.0, + "30": 2680509952.0, + "31": 2680509952.0, + "32": 2680509952.0, + "33": 2680509952.0, + "34": 2680509952.0, + "35": 2680509952.0, + "36": 2680509952.0, + "37": 2680509952.0, + "38": 2680509952.0, + "39": 2680509952.0, + "40": 2680509952.0, + "41": 2680509952.0, + "42": 2680509952.0, + "43": 2680509952.0, + "44": 2680509952.0, + "45": 2680511488.0, + "46": 2680511488.0, + "47": 2680511488.0, + "48": 2680511488.0, + "49": 2680511488.0, + "50": 2680511488.0, + "51": 2680511488.0, + "52": 2680511488.0, + "53": 2680511488.0, + "54": 2680511488.0, + "55": 2680511488.0, + "56": 2680511488.0, + "57": 2680511488.0, + "58": 2680511488.0, + "59": 2680511488.0, + "60": 2680511488.0, + "61": 2680511488.0, + "62": 2680511488.0, + "63": 2680511488.0, + "64": 2680511488.0, + "65": 2680511488.0, + "66": 2680511488.0, + "67": 2680511488.0, + "68": 2680511488.0, + "69": 2680511488.0, + "70": 2680511488.0, + "71": 2680511488.0, + "72": 2680511488.0, + "73": 2680511488.0, + "74": 2680511488.0, + "75": 2680511488.0, + "76": 2680511488.0, + "77": 2680511488.0, + "78": 2680511488.0, + "79": 2680511488.0, + "80": 2680511488.0, + "81": 2680511488.0, + "82": 2680511488.0, + "83": 2680511488.0, + "84": 2680511488.0, + "85": 2680511488.0, + "86": 2680511488.0, + "87": 2680511488.0, + "88": 2680511488.0, + "89": 2680511488.0, + "90": 2680511488.0, + "91": 2680511488.0, + "92": 2680511488.0, + "93": 2680511488.0, + "94": 2680511488.0, + "95": 2680511488.0, + "96": 2680511488.0, + "97": 2680511488.0, + "98": 2680511488.0, + "99": 2680511488.0, + "100": 2680511488.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.46829, - "3": 2.19116, - "4": 1.95809, - "5": 1.40068, - "6": 1.74239, - "7": 1.78913, - "8": 2.40733, - "9": 2.03666, - "10": 1.41436, - "11": 1.81047, - "12": 1.54887, - "13": 1.6746, - "14": 1.41729, - "15": 1.49013, - "16": 1.65749, - "17": 2.31569, - "18": 1.48178, - "19": 1.74017, - "20": 1.59272, - "21": 1.66007, - "22": 1.18588, - "23": 1.65601, - "24": 1.46353, - "25": 2.08613, - "26": 2.0327, - "27": 1.58821, - "28": 1.55791, - "29": 1.31213, - "30": 1.58975, - "31": 1.5941, - "32": 2.52529, - "33": 1.30473, - "34": 1.55279, - "35": 1.88412, - "36": 1.68746, - "37": 1.61395, - "38": 1.53528, - "39": 1.87614, - "40": 1.51385, - "41": 1.63358, - "42": 2.17455, - "43": 1.85386, - "44": 1.75152, - "45": 1.99743, - "46": 1.99147, - "47": 1.67234, - "48": 1.79448, - "49": 1.33978, - "50": 1.72926, - "51": 1.60723, - "52": 1.86053, - "53": 1.67615, - "54": 1.65761, - "55": 1.60302, - "56": 2.41491, - "57": 1.29123, - "58": 1.91886, - "59": 1.50095, - "60": 1.44339, - "61": 1.91511, - "62": 2.41539, - "63": 1.53165, - "64": 2.01962, - "65": 1.96493, - "66": 1.97869, - "67": 1.71408, - "68": 1.48955, - "69": 1.54863, - "70": 1.59482, - "71": 2.13789, - "72": 1.64486, - "73": 1.55983, - "74": 1.65312, - "75": 1.58677, - "76": 1.7833, - "77": 1.67262, - "78": 1.81241, - "79": 2.13938, - "80": 1.6973, - "81": 2.00145, - "82": 1.95043, - "83": 1.66526, - "84": 1.38491, - "85": 1.75651, - "86": 1.57279, - "87": 1.96625, - "88": 2.06727, - "89": 1.736, - "90": 1.65592, - "91": 1.68149, - "92": 2.15955, - "93": 1.51492, - "94": 1.73084, - "95": 2.00253, - "96": 2.01546, - "97": 1.7338, - "98": 1.93187, - "99": 2.28739, - "100": 1.55635 + "2": 6.67523, + "3": 2.06258, + "4": 1.69953, + "5": 1.37636, + "6": 1.45199, + "7": 1.48801, + "8": 2.00408, + "9": 1.90586, + "10": 1.49686, + "11": 1.75814, + "12": 1.44511, + "13": 1.5262, + "14": 1.3884, + "15": 1.40922, + "16": 1.51873, + "17": 2.08418, + "18": 1.48882, + "19": 1.80854, + "20": 1.46679, + "21": 1.68089, + "22": 1.29034, + "23": 1.9567, + "24": 1.40418, + "25": 1.68643, + "26": 1.89045, + "27": 1.62105, + "28": 1.46716, + "29": 1.37758, + "30": 1.65639, + "31": 1.35438, + "32": 2.22243, + "33": 1.31992, + "34": 1.66874, + "35": 1.779, + "36": 1.74745, + "37": 1.30926, + "38": 1.3221, + "39": 1.92388, + "40": 1.36222, + "41": 1.61109, + "42": 2.0811, + "43": 1.66098, + "44": 1.42944, + "45": 2.22627, + "46": 1.34745, + "47": 1.68305, + "48": 1.69366, + "49": 1.33326, + "50": 1.95139, + "51": 1.17275, + "52": 1.64993, + "53": 1.78943, + "54": 1.8485, + "55": 1.62059, + "56": 2.05431, + "57": 1.15401, + "58": 1.70575, + "59": 1.32599, + "60": 1.09194, + "61": 1.64023, + "62": 1.71365, + "63": 1.3522, + "64": 1.31334, + "65": 1.36082, + "66": 1.22388, + "67": 1.35303, + "68": 1.0984, + "69": 1.36163, + "70": 1.36761, + "71": 1.53159, + "72": 1.04617, + "73": 1.58807, + "74": 1.13336, + "75": 1.26459, + "76": 1.66113, + "77": 1.62282, + "78": 1.48725, + "79": 2.011, + "80": 1.50166, + "81": 2.01033, + "82": 1.73313, + "83": 1.39432, + "84": 1.42404, + "85": 1.48232, + "86": 1.39966, + "87": 2.0089, + "88": 1.9105, + "89": 1.96775, + "90": 1.51383, + "91": 1.50386, + "92": 2.08825, + "93": 1.40915, + "94": 1.66709, + "95": 1.71928, + "96": 1.83052, + "97": 1.42742, + "98": 1.60533, + "99": 2.1575, + "100": 1.41361 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json index 9349fc45786..87661cdebf6 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94135, + "1": 10.94145, "2": 10.93474, - "3": 10.94379, - "4": 10.93302, - "5": 10.93416, - "6": 10.93731, - "7": 10.93993, - "8": 10.93567, - "9": 10.93345, - "10": 10.93259, + "3": 10.94378, + "4": 10.93299, + "5": 10.93415, + "6": 10.9373, + "7": 10.93992, + "8": 10.93571, + "9": 10.93336, + "10": 10.93261, "11": 10.92506, - "12": 10.91971, + "12": 10.91979, "13": 10.92588, - "14": 10.92627, - "15": 10.8888, - "16": 10.87344, - "17": 10.87977, - "18": 10.88077, - "19": 10.86265, - "20": 10.80654, - "21": 10.78325, - "22": 10.784, - "23": 10.77457, - "24": 10.75095, - "25": 10.7431, - "26": 10.72415, - "27": 10.67536, - "28": 10.60819, - "29": 10.59235, - "30": 10.57838, - "31": 10.573, - "32": 10.54372, - "33": 10.52272, - "34": 10.50051, - "35": 10.48199, - "36": 10.471, - "37": 10.44151, + "14": 10.9262, + "15": 10.88882, + "16": 10.87345, + "17": 10.87979, + "18": 10.88079, + "19": 10.86266, + "20": 10.80657, + "21": 10.78329, + "22": 10.78397, + "23": 10.77461, + "24": 10.75087, + "25": 10.74315, + "26": 10.72411, + "27": 10.67539, + "28": 10.60823, + "29": 10.59236, + "30": 10.57847, + "31": 10.57301, + "32": 10.54375, + "33": 10.52277, + "34": 10.50056, + "35": 10.48201, + "36": 10.47109, + "37": 10.44148, "38": 10.4108, - "39": 10.39247, - "40": 10.38024, - "41": 10.37505, - "42": 10.3336, - "43": 10.32979, + "39": 10.39246, + "40": 10.38027, + "41": 10.37514, + "42": 10.33363, + "43": 10.32989, "44": 10.27977, - "45": 10.2955, - "46": 10.27386, + "45": 10.29556, + "46": 10.27396, "47": 10.26477, - "48": 10.23709, - "49": 10.19637, - "50": 10.17891 + "48": 10.23704, + "49": 10.19644, + "50": 10.17897 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1304.0, - "2": 1321.0, - "3": 1205.0, - "4": 1267.0, - "5": 1308.0, - "6": 1342.0, - "7": 1261.0, - "8": 1430.0, - "9": 1254.0, - "10": 1155.0, - "11": 1324.0, - "12": 1186.0, - "13": 1197.0, - "14": 1338.0, - "15": 1193.0, - "16": 1187.0, - "17": 1187.0, - "18": 1226.0, - "19": 1225.0, - "20": 1212.0, - "21": 1289.0, - "22": 1327.0, - "23": 1249.0, - "24": 1198.0, - "25": 1367.0, - "26": 1317.0, - "27": 1307.0, - "28": 1397.0, - "29": 1349.0, - "30": 1446.0, - "31": 1446.0, - "32": 1403.0, - "33": 1387.0, - "34": 1371.0, - "35": 1552.0, - "36": 1482.0, - "37": 1483.0, - "38": 1529.0, - "39": 1569.0, - "40": 1592.0, - "41": 1447.0, - "42": 1530.0, - "43": 1664.0, - "44": 1588.0, - "45": 1607.0, - "46": 1866.0, - "47": 1756.0, - "48": 1881.0, - "49": 1787.0, - "50": 1847.0 + "1": 1256.0, + "2": 1260.0, + "3": 1227.0, + "4": 1248.0, + "5": 1225.0, + "6": 1348.0, + "7": 1246.0, + "8": 1272.0, + "9": 1245.0, + "10": 1241.0, + "11": 1328.0, + "12": 1243.0, + "13": 1211.0, + "14": 1330.0, + "15": 1234.0, + "16": 1132.0, + "17": 1241.0, + "18": 1251.0, + "19": 1188.0, + "20": 1189.0, + "21": 1282.0, + "22": 1305.0, + "23": 1306.0, + "24": 1262.0, + "25": 1346.0, + "26": 1250.0, + "27": 1269.0, + "28": 1374.0, + "29": 1347.0, + "30": 1491.0, + "31": 1418.0, + "32": 1392.0, + "33": 1405.0, + "34": 1328.0, + "35": 1459.0, + "36": 1515.0, + "37": 1541.0, + "38": 1513.0, + "39": 1637.0, + "40": 1569.0, + "41": 1522.0, + "42": 1628.0, + "43": 1700.0, + "44": 1590.0, + "45": 1629.0, + "46": 1824.0, + "47": 1771.0, + "48": 1863.0, + "49": 1725.0, + "50": 1864.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.97532, - "3": 2.47004, - "4": 1.72283, - "5": 2.23995, - "6": 2.20475, - "7": 2.30786, - "8": 2.66458, - "9": 1.91374, - "10": 2.01855, - "11": 2.45783, - "12": 1.7446, - "13": 2.14912, - "14": 2.64415, - "15": 1.38891, - "16": 2.13496, - "17": 2.09733, - "18": 1.78824, - "19": 1.99043, - "20": 2.11111, - "21": 2.2353, - "22": 1.86339, - "23": 2.0786, - "24": 1.60516, - "25": 1.88296, - "26": 2.61021, - "27": 1.49203, - "28": 2.18545, - "29": 2.23402, - "30": 1.77951, - "31": 1.89761, - "32": 2.27256, - "33": 2.06849, - "34": 2.99019, - "35": 2.22115, - "36": 2.30154, - "37": 2.13759, - "38": 2.36976, - "39": 2.08565, - "40": 2.42789, - "41": 1.9744, - "42": 1.98984, - "43": 2.0876, - "44": 1.94941, - "45": 2.94672, - "46": 1.81504, - "47": 2.30482, - "48": 1.73951, - "49": 2.55665, - "50": 2.48619 + "2": 6.58833, + "3": 2.82726, + "4": 2.32626, + "5": 2.98593, + "6": 2.70855, + "7": 3.03962, + "8": 3.10052, + "9": 2.35609, + "10": 2.40017, + "11": 2.8318, + "12": 2.34585, + "13": 2.45915, + "14": 3.12941, + "15": 2.2094, + "16": 2.6191, + "17": 2.84693, + "18": 2.6863, + "19": 2.34742, + "20": 2.88449, + "21": 3.2294, + "22": 2.59703, + "23": 2.69418, + "24": 2.25625, + "25": 2.27442, + "26": 3.39863, + "27": 1.86494, + "28": 2.70505, + "29": 2.64378, + "30": 2.31387, + "31": 2.65588, + "32": 2.8403, + "33": 2.52734, + "34": 3.55006, + "35": 2.44245, + "36": 2.69118, + "37": 2.1851, + "38": 2.99974, + "39": 2.32277, + "40": 2.32431, + "41": 2.7405, + "42": 2.13953, + "43": 2.65826, + "44": 2.51167, + "45": 3.03814, + "46": 2.35204, + "47": 2.37476, + "48": 2.44441, + "49": 2.82683, + "50": 2.80111 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json index 846508e1171..814a6d81d20 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92144, - "2": 10.91663, - "3": 10.92315, + "1": 10.9215, + "2": 10.91662, + "3": 10.92316, "4": 10.92349, - "5": 10.91368, - "6": 10.91998, - "7": 10.91916, - "8": 10.91328, - "9": 10.91111, - "10": 10.91012, - "11": 10.8989, - "12": 10.91037, + "5": 10.91361, + "6": 10.91995, + "7": 10.91914, + "8": 10.91332, + "9": 10.91108, + "10": 10.91007, + "11": 10.899, + "12": 10.91034, "13": 10.90061, - "14": 10.89943, - "15": 10.86488, - "16": 10.86595, - "17": 10.86291, - "18": 10.85837, - "19": 10.85573, - "20": 10.7754, - "21": 10.76241, + "14": 10.89941, + "15": 10.86497, + "16": 10.86604, + "17": 10.86294, + "18": 10.85832, + "19": 10.85569, + "20": 10.77536, + "21": 10.76245, "22": 10.74117, - "23": 10.73079, - "24": 10.71913, - "25": 10.69554, - "26": 10.68545, - "27": 10.63923, - "28": 10.5801, - "29": 10.54482, - "30": 10.50512, - "31": 10.50885, - "32": 10.49609, - "33": 10.44579, - "34": 10.42757, - "35": 10.42699, - "36": 10.39309, - "37": 10.37167, - "38": 10.36409, - "39": 10.33254, - "40": 10.30544, - "41": 10.29046, - "42": 10.25657, - "43": 10.24685, - "44": 10.21515, - "45": 10.22572, + "23": 10.73085, + "24": 10.71917, + "25": 10.69555, + "26": 10.68556, + "27": 10.63915, + "28": 10.58018, + "29": 10.54488, + "30": 10.50515, + "31": 10.50891, + "32": 10.49613, + "33": 10.44581, + "34": 10.42758, + "35": 10.42707, + "36": 10.39311, + "37": 10.37175, + "38": 10.36411, + "39": 10.33261, + "40": 10.30549, + "41": 10.29042, + "42": 10.25659, + "43": 10.24688, + "44": 10.2152, + "45": 10.22577, "46": 10.17113, - "47": 10.17396, - "48": 10.13505, - "49": 10.13403, - "50": 10.11799 + "47": 10.17395, + "48": 10.13514, + "49": 10.13406, + "50": 10.11802 } }, "num-zeros": { @@ -62,55 +62,55 @@ "step_interval": 1, "values": { "1": 22962624.0, - "2": 22849806.0, - "3": 22709408.0, - "4": 22777744.0, - "5": 22783288.0, - "6": 22745536.0, - "7": 22871092.0, - "8": 22611466.0, - "9": 22758692.0, - "10": 22489228.0, - "11": 22753938.0, - "12": 22640896.0, - "13": 23322980.0, - "14": 22992186.0, - "15": 22718256.0, - "16": 22821932.0, - "17": 22928904.0, - "18": 22994992.0, - "19": 23084580.0, - "20": 22723010.0, - "21": 22917944.0, - "22": 22945048.0, - "23": 22628692.0, - "24": 22862560.0, - "25": 22636388.0, - "26": 23004176.0, - "27": 22801668.0, - "28": 22999066.0, - "29": 22978864.0, - "30": 22943404.0, - "31": 22908058.0, + "2": 22849912.0, + "3": 22709392.0, + "4": 22777726.0, + "5": 22783112.0, + "6": 22745530.0, + "7": 22871044.0, + "8": 22611470.0, + "9": 22758602.0, + "10": 22489074.0, + "11": 22753998.0, + "12": 22640984.0, + "13": 23323030.0, + "14": 22992300.0, + "15": 22718184.0, + "16": 22822000.0, + "17": 22929068.0, + "18": 22995004.0, + "19": 23084684.0, + "20": 22722980.0, + "21": 22917956.0, + "22": 22945032.0, + "23": 22628616.0, + "24": 22862492.0, + "25": 22636248.0, + "26": 23004286.0, + "27": 22801714.0, + "28": 22998980.0, + "29": 22978820.0, + "30": 22943460.0, + "31": 22908188.0, "32": 22665440.0, - "33": 22740520.0, - "34": 23076388.0, - "35": 22750142.0, - "36": 22694848.0, - "37": 23103828.0, - "38": 22964694.0, - "39": 22985412.0, - "40": 22750684.0, - "41": 23064592.0, - "42": 22688200.0, - "43": 22987774.0, - "44": 22704794.0, - "45": 22846540.0, - "46": 22733852.0, - "47": 22849936.0, - "48": 22833196.0, - "49": 22889180.0, - "50": 22645714.0 + "33": 22740688.0, + "34": 23076412.0, + "35": 22750120.0, + "36": 22694804.0, + "37": 23103692.0, + "38": 22964744.0, + "39": 22985376.0, + "40": 22750762.0, + "41": 23064708.0, + "42": 22688340.0, + "43": 22987748.0, + "44": 22704870.0, + "45": 22846532.0, + "46": 22733792.0, + "47": 22849980.0, + "48": 22833180.0, + "49": 22889156.0, + "50": 22645686.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 688128512.0, - "2": 688128512.0, - "3": 688128512.0, - "4": 688128512.0, - "5": 688128512.0, - "6": 688128512.0, - "7": 688128512.0, - "8": 688128512.0, - "9": 688128512.0, - "10": 688128512.0, - "11": 688128512.0, - "12": 688128512.0, - "13": 688128512.0, - "14": 688128512.0, - "15": 688128512.0, - "16": 688128512.0, - "17": 688128512.0, - "18": 688128512.0, - "19": 688128512.0, - "20": 688128512.0, - "21": 688128512.0, - "22": 688128512.0, - "23": 688128512.0, - "24": 688128512.0, - "25": 688128512.0, - "26": 688128512.0, - "27": 688128512.0, - "28": 688128512.0, - "29": 688128512.0, - "30": 688128512.0, - "31": 688128512.0, - "32": 688128512.0, - "33": 688128512.0, - "34": 688128512.0, - "35": 688128512.0, - "36": 688128512.0, - "37": 688128512.0, - "38": 688128512.0, - "39": 688128512.0, - "40": 688128512.0, - "41": 688128512.0, - "42": 688128512.0, - "43": 688128512.0, - "44": 688128512.0, - "45": 688128512.0, - "46": 688128512.0, - "47": 688128512.0, - "48": 688128512.0, - "49": 688128512.0, - "50": 688128512.0 + "1": 689177088.0, + "2": 689177088.0, + "3": 689177088.0, + "4": 689177088.0, + "5": 689177088.0, + "6": 689177088.0, + "7": 689177088.0, + "8": 689177088.0, + "9": 689177088.0, + "10": 689177088.0, + "11": 689177088.0, + "12": 689177088.0, + "13": 689177088.0, + "14": 689177088.0, + "15": 689177088.0, + "16": 689177088.0, + "17": 689177088.0, + "18": 689177088.0, + "19": 689177088.0, + "20": 689177088.0, + "21": 689177088.0, + "22": 689177088.0, + "23": 689177088.0, + "24": 689177088.0, + "25": 689177088.0, + "26": 689177088.0, + "27": 689177088.0, + "28": 689177088.0, + "29": 689177088.0, + "30": 689177088.0, + "31": 689177088.0, + "32": 689177088.0, + "33": 689177088.0, + "34": 689177088.0, + "35": 689177088.0, + "36": 689177088.0, + "37": 689177088.0, + "38": 689177088.0, + "39": 689177088.0, + "40": 689177088.0, + "41": 689177088.0, + "42": 689177088.0, + "43": 689177088.0, + "44": 689177088.0, + "45": 689177088.0, + "46": 689177088.0, + "47": 689177088.0, + "48": 689177088.0, + "49": 689177088.0, + "50": 689177088.0 } }, "mem-max-allocated-bytes": { @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 2158025216.0, - "2": 2416613888.0, - "3": 2416613888.0, - "4": 2416613888.0, - "5": 2416613888.0, - "6": 2416613888.0, - "7": 2416613888.0, - "8": 2416613888.0, - "9": 2416613888.0, - "10": 2416613888.0, - "11": 2416613888.0, - "12": 2416613888.0, - "13": 2416613888.0, - "14": 2416613888.0, - "15": 2416613888.0, - "16": 2416613888.0, - "17": 2416613888.0, - "18": 2416613888.0, - "19": 2416613888.0, - "20": 2416613888.0, - "21": 2416613888.0, - "22": 2416613888.0, - "23": 2416613888.0, - "24": 2416613888.0, - "25": 2416613888.0, - "26": 2416613888.0, - "27": 2416613888.0, - "28": 2416613888.0, - "29": 2416613888.0, - "30": 2416613888.0, - "31": 2416613888.0, - "32": 2416613888.0, - "33": 2416613888.0, - "34": 2416613888.0, - "35": 2416613888.0, - "36": 2416613888.0, - "37": 2416613888.0, - "38": 2416613888.0, - "39": 2416613888.0, - "40": 2416613888.0, - "41": 2416613888.0, - "42": 2416613888.0, - "43": 2416613888.0, - "44": 2416613888.0, - "45": 2416613888.0, - "46": 2416613888.0, - "47": 2416613888.0, - "48": 2416613888.0, - "49": 2416613888.0, - "50": 2416613888.0 + "2": 2416614912.0, + "3": 2416614912.0, + "4": 2416614912.0, + "5": 2416614912.0, + "6": 2416614912.0, + "7": 2416614912.0, + "8": 2416614912.0, + "9": 2416614912.0, + "10": 2416614912.0, + "11": 2416614912.0, + "12": 2416614912.0, + "13": 2416614912.0, + "14": 2416614912.0, + "15": 2416614912.0, + "16": 2416614912.0, + "17": 2416614912.0, + "18": 2416614912.0, + "19": 2416614912.0, + "20": 2416614912.0, + "21": 2416614912.0, + "22": 2416614912.0, + "23": 2416614912.0, + "24": 2416614912.0, + "25": 2416614912.0, + "26": 2416614912.0, + "27": 2416614912.0, + "28": 2416614912.0, + "29": 2416614912.0, + "30": 2416614912.0, + "31": 2416614912.0, + "32": 2416614912.0, + "33": 2416614912.0, + "34": 2416614912.0, + "35": 2416614912.0, + "36": 2416614912.0, + "37": 2416614912.0, + "38": 2416614912.0, + "39": 2416614912.0, + "40": 2416614912.0, + "41": 2416614912.0, + "42": 2416614912.0, + "43": 2416614912.0, + "44": 2416614912.0, + "45": 2416614912.0, + "46": 2416614912.0, + "47": 2416614912.0, + "48": 2416614912.0, + "49": 2416614912.0, + "50": 2416614912.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.78477, - "3": 0.16468, - "4": 0.15003, - "5": 0.14868, - "6": 0.14939, - "7": 0.14914, - "8": 0.14951, - "9": 0.15008, - "10": 0.15137, - "11": 0.15025, - "12": 0.14989, - "13": 0.15049, - "14": 0.15141, - "15": 0.14926, - "16": 0.14964, - "17": 0.15013, - "18": 0.14973, - "19": 0.15094, - "20": 0.1502, - "21": 0.14906, - "22": 0.14889, - "23": 0.1496, - "24": 0.14992, - "25": 0.15151, - "26": 0.15242, - "27": 0.15087, - "28": 0.14993, - "29": 0.15025, - "30": 0.15021, - "31": 0.15012, - "32": 0.14929, - "33": 0.15022, - "34": 0.15092, - "35": 0.14984, - "36": 0.14972, - "37": 0.14984, - "38": 0.15009, - "39": 0.14959, - "40": 0.15051, - "41": 0.15013, - "42": 0.15049, - "43": 0.15046, - "44": 0.15125, - "45": 0.15053, - "46": 0.15064, - "47": 0.15115, - "48": 0.15073, - "49": 0.151, - "50": 0.15042 + "2": 8.65565, + "3": 0.17811, + "4": 0.12632, + "5": 0.12736, + "6": 0.12552, + "7": 0.12849, + "8": 0.12677, + "9": 0.12467, + "10": 0.12625, + "11": 0.1254, + "12": 0.12305, + "13": 0.12669, + "14": 0.16671, + "15": 0.12727, + "16": 0.12541, + "17": 0.23956, + "18": 0.1278, + "19": 0.12656, + "20": 0.12712, + "21": 0.12366, + "22": 0.1229, + "23": 0.12436, + "24": 0.12416, + "25": 0.12402, + "26": 0.1234, + "27": 0.12449, + "28": 0.12503, + "29": 0.12301, + "30": 0.12423, + "31": 0.12439, + "32": 0.12374, + "33": 0.12579, + "34": 0.23684, + "35": 0.12629, + "36": 0.12499, + "37": 0.12506, + "38": 0.12436, + "39": 0.12644, + "40": 0.12645, + "41": 0.12803, + "42": 0.12686, + "43": 0.12418, + "44": 0.12489, + "45": 0.12345, + "46": 0.12442, + "47": 0.12536, + "48": 0.12359, + "49": 0.24403, + "50": 0.1231 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json index fe6479caf7e..6d8d429d0f2 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89814, - "2": 10.89631, - "3": 10.90187, - "4": 10.89348, - "5": 10.89713, - "6": 10.88447, - "7": 10.90724, - "8": 10.8899, - "9": 10.90482, - "10": 10.89557, - "11": 10.88482, - "12": 10.87805, - "13": 10.89171, - "14": 10.87219, + "1": 10.8982, + "2": 10.89629, + "3": 10.90191, + "4": 10.89353, + "5": 10.89714, + "6": 10.88452, + "7": 10.90719, + "8": 10.88997, + "9": 10.90487, + "10": 10.89562, + "11": 10.88494, + "12": 10.87811, + "13": 10.89176, + "14": 10.87214, "15": 10.85219, - "16": 10.84007, - "17": 10.85267, - "18": 10.83214, - "19": 10.83213, - "20": 10.74658, - "21": 10.74631, - "22": 10.7218, - "23": 10.70891, - "24": 10.68202, - "25": 10.67115, - "26": 10.66267, - "27": 10.60776, - "28": 10.55958, - "29": 10.52531, - "30": 10.49467, - "31": 10.4859, - "32": 10.472, - "33": 10.42622, - "34": 10.41733, - "35": 10.40437, - "36": 10.37776, - "37": 10.34605, - "38": 10.35576, - "39": 10.31893, - "40": 10.30851, + "16": 10.84008, + "17": 10.85268, + "18": 10.83216, + "19": 10.83211, + "20": 10.74654, + "21": 10.74629, + "22": 10.72187, + "23": 10.709, + "24": 10.68205, + "25": 10.67122, + "26": 10.66272, + "27": 10.60786, + "28": 10.55965, + "29": 10.52532, + "30": 10.49471, + "31": 10.48597, + "32": 10.47209, + "33": 10.42627, + "34": 10.41738, + "35": 10.40438, + "36": 10.37779, + "37": 10.34612, + "38": 10.35579, + "39": 10.31902, + "40": 10.30856, "41": 10.28993, - "42": 10.24612, - "43": 10.23762, - "44": 10.21154, - "45": 10.22062, - "46": 10.17855, + "42": 10.24616, + "43": 10.23767, + "44": 10.21156, + "45": 10.22066, + "46": 10.1786, "47": 10.17787, - "48": 10.12575, - "49": 10.12681, - "50": 10.1219 + "48": 10.1258, + "49": 10.1268, + "50": 10.12202 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22962392.0, - "2": 22849530.0, - "3": 22709824.0, - "4": 22777500.0, - "5": 22782868.0, - "6": 22745208.0, - "7": 22871616.0, - "8": 22611016.0, - "9": 22758184.0, - "10": 22488768.0, - "11": 22753326.0, - "12": 22640850.0, - "13": 23323068.0, - "14": 22992126.0, - "15": 22717790.0, - "16": 22822396.0, - "17": 22928836.0, - "18": 22994884.0, - "19": 23085022.0, - "20": 22723130.0, - "21": 22917994.0, - "22": 22944704.0, - "23": 22628748.0, - "24": 22862012.0, - "25": 22636090.0, - "26": 23004312.0, - "27": 22801808.0, - "28": 22998904.0, - "29": 22978984.0, - "30": 22943914.0, - "31": 22908222.0, - "32": 22665164.0, - "33": 22740194.0, - "34": 23076468.0, - "35": 22749952.0, - "36": 22694654.0, - "37": 23103940.0, - "38": 22964598.0, - "39": 22985086.0, - "40": 22750600.0, - "41": 23064904.0, - "42": 22688808.0, - "43": 22987284.0, - "44": 22704616.0, - "45": 22847632.0, - "46": 22733968.0, - "47": 22850384.0, - "48": 22833474.0, - "49": 22888972.0, - "50": 22644660.0 + "1": 22962456.0, + "2": 22849516.0, + "3": 22709852.0, + "4": 22777460.0, + "5": 22782948.0, + "6": 22745084.0, + "7": 22871700.0, + "8": 22610966.0, + "9": 22758244.0, + "10": 22488846.0, + "11": 22753404.0, + "12": 22640832.0, + "13": 23323176.0, + "14": 22991992.0, + "15": 22717880.0, + "16": 22822390.0, + "17": 22928926.0, + "18": 22994824.0, + "19": 23085090.0, + "20": 22723148.0, + "21": 22917848.0, + "22": 22944810.0, + "23": 22628814.0, + "24": 22861948.0, + "25": 22636144.0, + "26": 23004472.0, + "27": 22801852.0, + "28": 22999004.0, + "29": 22978964.0, + "30": 22943972.0, + "31": 22908184.0, + "32": 22665242.0, + "33": 22740176.0, + "34": 23076434.0, + "35": 22749892.0, + "36": 22694802.0, + "37": 23104044.0, + "38": 22964560.0, + "39": 22985088.0, + "40": 22750652.0, + "41": 23065016.0, + "42": 22688756.0, + "43": 22987280.0, + "44": 22704512.0, + "45": 22847482.0, + "46": 22733950.0, + "47": 22850374.0, + "48": 22833462.0, + "49": 22889040.0, + "50": 22644900.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.48655, - "3": 0.14271, - "4": 0.13967, - "5": 0.13987, - "6": 0.14039, - "7": 0.14008, - "8": 0.14339, - "9": 0.1387, - "10": 0.14028, - "11": 0.13933, - "12": 0.14022, - "13": 0.14075, - "14": 0.13872, - "15": 0.14042, - "16": 0.14185, - "17": 0.1402, - "18": 0.14062, - "19": 0.13885, - "20": 0.13913, - "21": 0.13948, - "22": 0.13888, - "23": 0.13805, - "24": 0.13946, - "25": 0.14414, - "26": 0.13982, - "27": 0.13865, - "28": 0.14095, - "29": 0.13879, - "30": 0.13921, - "31": 0.14151, - "32": 0.14307, - "33": 0.13838, - "34": 0.1398, - "35": 0.13938, - "36": 0.14126, - "37": 0.14155, - "38": 0.13902, - "39": 0.1409, - "40": 0.14029, - "41": 0.14115, - "42": 0.14031, - "43": 0.14074, - "44": 0.13928, - "45": 0.13869, - "46": 0.13901, - "47": 0.14205, - "48": 0.14117, - "49": 0.14874, - "50": 0.13916 + "2": 7.0123, + "3": 2.10027, + "4": 2.37642, + "5": 0.78008, + "6": 1.8514, + "7": 1.72565, + "8": 1.6157, + "9": 1.97865, + "10": 0.77783, + "11": 1.41285, + "12": 1.33393, + "13": 1.29193, + "14": 1.65314, + "15": 0.89258, + "16": 1.27735, + "17": 1.6288, + "18": 1.22678, + "19": 1.20554, + "20": 1.52055, + "21": 1.5876, + "22": 1.11762, + "23": 1.71869, + "24": 1.67747, + "25": 0.80162, + "26": 1.28077, + "27": 1.40005, + "28": 1.41046, + "29": 1.34902, + "30": 1.15994, + "31": 1.39814, + "32": 2.10905, + "33": 1.97766, + "34": 0.66823, + "35": 2.304, + "36": 1.66806, + "37": 1.41602, + "38": 1.13389, + "39": 2.32961, + "40": 2.65832, + "41": 1.70809, + "42": 2.3591, + "43": 1.83825, + "44": 1.99452, + "45": 1.82255, + "46": 1.37194, + "47": 2.63984, + "48": 2.29745, + "49": 1.62156, + "50": 1.81521 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json index 24423fd16b1..893076c2415 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.91778, - "2": 10.91602, - "3": 10.9198, + "1": 10.91788, + "2": 10.91609, + "3": 10.91977, "4": 10.91999, - "5": 10.91201, - "6": 10.91954, - "7": 10.92034, - "8": 10.91471, - "9": 10.90787, - "10": 10.90661, + "5": 10.91193, + "6": 10.91967, + "7": 10.92039, + "8": 10.91461, + "9": 10.90788, + "10": 10.90662, "11": 10.90292, - "12": 10.90754, - "13": 10.90318, - "14": 10.8975, - "15": 10.86511, - "16": 10.86752, - "17": 10.8629, - "18": 10.85425, - "19": 10.85849, - "20": 10.77184, - "21": 10.76543, - "22": 10.74101, - "23": 10.73028, - "24": 10.71854, - "25": 10.69262, - "26": 10.68889, - "27": 10.63789, - "28": 10.57948, + "12": 10.90752, + "13": 10.90331, + "14": 10.89747, + "15": 10.86508, + "16": 10.86748, + "17": 10.86291, + "18": 10.85426, + "19": 10.85851, + "20": 10.77185, + "21": 10.7654, + "22": 10.741, + "23": 10.73033, + "24": 10.7186, + "25": 10.69268, + "26": 10.68897, + "27": 10.63802, + "28": 10.57955, "29": 10.54148, - "30": 10.50449, - "31": 10.51059, - "32": 10.4944, - "33": 10.4464, - "34": 10.4268, - "35": 10.42715, - "36": 10.39378, - "37": 10.37391, - "38": 10.36201, - "39": 10.33359, - "40": 10.30551, - "41": 10.29099, - "42": 10.25817, - "43": 10.24905, - "44": 10.21682, - "45": 10.227, - "46": 10.17061, - "47": 10.17437, - "48": 10.13475, - "49": 10.13473, - "50": 10.11788 + "30": 10.50454, + "31": 10.51064, + "32": 10.49445, + "33": 10.44646, + "34": 10.42685, + "35": 10.42724, + "36": 10.39377, + "37": 10.37397, + "38": 10.362, + "39": 10.3337, + "40": 10.30562, + "41": 10.29104, + "42": 10.25824, + "43": 10.24913, + "44": 10.21691, + "45": 10.22705, + "46": 10.17066, + "47": 10.17441, + "48": 10.13478, + "49": 10.13485, + "50": 10.11789 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22961698.0, - "2": 22850648.0, - "3": 22708944.0, - "4": 22777238.0, - "5": 22782670.0, - "6": 22745544.0, - "7": 22871068.0, - "8": 22611328.0, - "9": 22758860.0, - "10": 22488844.0, - "11": 22752972.0, - "12": 22641496.0, - "13": 23322216.0, - "14": 22991336.0, - "15": 22717318.0, - "16": 22822488.0, - "17": 22929414.0, - "18": 22994556.0, - "19": 23084612.0, - "20": 22722388.0, - "21": 22918198.0, - "22": 22944376.0, - "23": 22628422.0, - "24": 22862476.0, - "25": 22636572.0, - "26": 23004492.0, - "27": 22801540.0, - "28": 22998012.0, - "29": 22979304.0, - "30": 22944624.0, - "31": 22907846.0, - "32": 22665480.0, - "33": 22739828.0, - "34": 23075384.0, - "35": 22750508.0, - "36": 22694996.0, - "37": 23103238.0, - "38": 22963552.0, - "39": 22984982.0, - "40": 22749680.0, - "41": 23065176.0, - "42": 22688580.0, - "43": 22987652.0, - "44": 22704676.0, - "45": 22847124.0, - "46": 22734064.0, - "47": 22849372.0, - "48": 22832772.0, - "49": 22888554.0, - "50": 22644322.0 + "1": 22961726.0, + "2": 22850560.0, + "3": 22708848.0, + "4": 22777128.0, + "5": 22782776.0, + "6": 22745572.0, + "7": 22871024.0, + "8": 22611400.0, + "9": 22758804.0, + "10": 22488848.0, + "11": 22753076.0, + "12": 22641586.0, + "13": 23322250.0, + "14": 22991348.0, + "15": 22717328.0, + "16": 22822440.0, + "17": 22929468.0, + "18": 22994548.0, + "19": 23084632.0, + "20": 22722448.0, + "21": 22918196.0, + "22": 22944310.0, + "23": 22628442.0, + "24": 22862544.0, + "25": 22636710.0, + "26": 23004344.0, + "27": 22801504.0, + "28": 22997960.0, + "29": 22979424.0, + "30": 22944576.0, + "31": 22907836.0, + "32": 22665460.0, + "33": 22739932.0, + "34": 23075416.0, + "35": 22750576.0, + "36": 22694956.0, + "37": 23103186.0, + "38": 22963456.0, + "39": 22985140.0, + "40": 22749690.0, + "41": 23065180.0, + "42": 22688576.0, + "43": 22987788.0, + "44": 22704648.0, + "45": 22847184.0, + "46": 22734016.0, + "47": 22849236.0, + "48": 22832806.0, + "49": 22888772.0, + "50": 22644324.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.00307, - "3": 2.46501, - "4": 1.71717, - "5": 1.50195, - "6": 2.0154, - "7": 1.88147, - "8": 2.40787, - "9": 2.39934, - "10": 2.18348, - "11": 2.62062, - "12": 1.9888, - "13": 3.01208, - "14": 3.6425, - "15": 4.10603, - "16": 3.39627, - "17": 4.04231, - "18": 3.21431, - "19": 2.64929, - "20": 2.68367, - "21": 3.2317, - "22": 2.82182, - "23": 3.19262, - "24": 2.79362, - "25": 3.71877, - "26": 4.15527, - "27": 2.79149, - "28": 2.64167, - "29": 1.72488, - "30": 2.23461, - "31": 2.14333, - "32": 3.0767, - "33": 1.77825, - "34": 1.7662, - "35": 2.65221, - "36": 2.36333, - "37": 2.34348, - "38": 1.63998, - "39": 2.5093, - "40": 2.17594, - "41": 2.00567, - "42": 2.69642, - "43": 2.00334, - "44": 2.09538, - "45": 2.99263, - "46": 2.10827, - "47": 2.39942, - "48": 2.93432, - "49": 2.07109, - "50": 2.32718 + "2": 6.12541, + "3": 2.74793, + "4": 2.31583, + "5": 1.74228, + "6": 1.9397, + "7": 2.00702, + "8": 2.2206, + "9": 2.35016, + "10": 2.05467, + "11": 2.10073, + "12": 1.83265, + "13": 1.38771, + "14": 1.98621, + "15": 1.52072, + "16": 1.49764, + "17": 1.95727, + "18": 1.49423, + "19": 1.87375, + "20": 1.56096, + "21": 1.81752, + "22": 1.43746, + "23": 2.41142, + "24": 1.47264, + "25": 2.14919, + "26": 1.97958, + "27": 1.63878, + "28": 1.37028, + "29": 1.62029, + "30": 1.64194, + "31": 1.32762, + "32": 2.57747, + "33": 1.30711, + "34": 1.47185, + "35": 2.07861, + "36": 1.87657, + "37": 1.67994, + "38": 1.13848, + "39": 1.97776, + "40": 2.01401, + "41": 1.72988, + "42": 2.17712, + "43": 1.75538, + "44": 1.97344, + "45": 2.22674, + "46": 1.77237, + "47": 2.00103, + "48": 2.3676, + "49": 1.41807, + "50": 2.00258 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json index 22edca3da6a..27dac8244a8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.90589, + "1": 10.90588, "2": 10.90691, - "3": 10.91424, - "4": 10.90501, - "5": 10.90407, - "6": 10.90341, - "7": 10.90457, - "8": 10.91075, - "9": 10.91462, - "10": 10.92225, - "11": 10.90376, - "12": 10.89844, - "13": 10.91112, - "14": 10.91108, - "15": 10.8922, - "16": 10.88768, - "17": 10.89575, - "18": 10.88501, - "19": 10.8964, - "20": 10.85142, - "21": 10.85551, - "22": 10.85362, - "23": 10.84217, - "24": 10.82684, - "25": 10.8278, - "26": 10.83356, - "27": 10.81541, - "28": 10.76128, - "29": 10.73223, + "3": 10.91428, + "4": 10.90504, + "5": 10.90412, + "6": 10.90343, + "7": 10.90465, + "8": 10.91074, + "9": 10.91457, + "10": 10.92218, + "11": 10.90378, + "12": 10.89843, + "13": 10.9111, + "14": 10.91099, + "15": 10.89224, + "16": 10.88763, + "17": 10.89572, + "18": 10.88498, + "19": 10.8963, + "20": 10.85135, + "21": 10.85567, + "22": 10.85363, + "23": 10.84219, + "24": 10.82685, + "25": 10.82787, + "26": 10.83361, + "27": 10.81551, + "28": 10.7613, + "29": 10.73228, "30": 10.71663, - "31": 10.70628, + "31": 10.70632, "32": 10.71514, - "33": 10.68543, - "34": 10.6639, - "35": 10.66054, - "36": 10.64658, - "37": 10.62828, - "38": 10.60639, - "39": 10.55284, - "40": 10.54237, - "41": 10.52912, - "42": 10.50959, - "43": 10.50121, - "44": 10.46489, - "45": 10.47702, - "46": 10.43281, - "47": 10.42993, - "48": 10.39104, - "49": 10.3875, - "50": 10.36641 + "33": 10.68552, + "34": 10.66402, + "35": 10.66049, + "36": 10.64661, + "37": 10.62829, + "38": 10.60641, + "39": 10.55291, + "40": 10.54242, + "41": 10.52918, + "42": 10.50971, + "43": 10.50117, + "44": 10.46494, + "45": 10.47707, + "46": 10.43283, + "47": 10.43, + "48": 10.39114, + "49": 10.3876, + "50": 10.36642 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22961264.0, - "2": 22849576.0, - "3": 22708588.0, - "4": 22776928.0, - "5": 22782980.0, - "6": 22744748.0, - "7": 22870716.0, - "8": 22610662.0, - "9": 22758436.0, - "10": 22489160.0, - "11": 22753590.0, - "12": 22640968.0, - "13": 23321324.0, - "14": 22991788.0, - "15": 22717966.0, - "16": 22821384.0, - "17": 22927734.0, - "18": 22994024.0, - "19": 23083938.0, - "20": 22722690.0, + "1": 22961156.0, + "2": 22849712.0, + "3": 22708660.0, + "4": 22776910.0, + "5": 22782816.0, + "6": 22744726.0, + "7": 22870724.0, + "8": 22610726.0, + "9": 22758372.0, + "10": 22489186.0, + "11": 22753572.0, + "12": 22640874.0, + "13": 23321416.0, + "14": 22991852.0, + "15": 22717996.0, + "16": 22821416.0, + "17": 22927816.0, + "18": 22994048.0, + "19": 23083936.0, + "20": 22722708.0, "21": 22917308.0, - "22": 22943996.0, - "23": 22628722.0, - "24": 22861472.0, - "25": 22635096.0, - "26": 23004660.0, - "27": 22800788.0, + "22": 22943936.0, + "23": 22628596.0, + "24": 22861458.0, + "25": 22635070.0, + "26": 23004610.0, + "27": 22800832.0, "28": 22998472.0, - "29": 22977780.0, - "30": 22943280.0, - "31": 22907248.0, - "32": 22665144.0, - "33": 22740060.0, - "34": 23074802.0, - "35": 22750212.0, - "36": 22694560.0, - "37": 23102744.0, - "38": 22964788.0, - "39": 22983556.0, - "40": 22748884.0, - "41": 23064224.0, - "42": 22687630.0, - "43": 22986804.0, - "44": 22703842.0, - "45": 22846044.0, - "46": 22732348.0, - "47": 22848390.0, - "48": 22832470.0, - "49": 22887488.0, - "50": 22643730.0 + "29": 22977832.0, + "30": 22943248.0, + "31": 22907210.0, + "32": 22665136.0, + "33": 22740110.0, + "34": 23074898.0, + "35": 22750240.0, + "36": 22694568.0, + "37": 23102684.0, + "38": 22964840.0, + "39": 22983522.0, + "40": 22748844.0, + "41": 23064160.0, + "42": 22687728.0, + "43": 22986872.0, + "44": 22703740.0, + "45": 22846034.0, + "46": 22732316.0, + "47": 22848236.0, + "48": 22832420.0, + "49": 22887538.0, + "50": 22643784.0 } }, "mem-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2610026496.0, + "1": 2610027008.0, "2": 2842349056.0, "3": 2842349056.0, "4": 2842349056.0, "5": 2842349056.0, "6": 2842349056.0, "7": 2842349056.0, - "8": 2843266560.0, - "9": 2843266560.0, - "10": 2843266560.0, - "11": 2843266560.0, - "12": 2843266560.0, - "13": 2843266560.0, - "14": 2843266560.0, - "15": 2843266560.0, - "16": 2843266560.0, - "17": 2843266560.0, - "18": 2843266560.0, - "19": 2843266560.0, - "20": 2843266560.0, - "21": 2843266560.0, - "22": 2843266560.0, - "23": 2843266560.0, - "24": 2843266560.0, - "25": 2843266560.0, - "26": 2843266560.0, - "27": 2843266560.0, - "28": 2843266560.0, - "29": 2843266560.0, - "30": 2843266560.0, - "31": 2843266560.0, - "32": 2843266560.0, - "33": 2843266560.0, - "34": 2843266560.0, - "35": 2843266560.0, - "36": 2843266560.0, - "37": 2843266560.0, - "38": 2843266560.0, - "39": 2843266560.0, - "40": 2843266560.0, - "41": 2843266560.0, - "42": 2843266560.0, - "43": 2843266560.0, - "44": 2843266560.0, - "45": 2843266560.0, - "46": 2843266560.0, - "47": 2843266560.0, - "48": 2843266560.0, - "49": 2843266560.0, - "50": 2843266560.0 + "8": 2842349056.0, + "9": 2842349056.0, + "10": 2842349056.0, + "11": 2842349056.0, + "12": 2842349056.0, + "13": 2842349056.0, + "14": 2842349056.0, + "15": 2842349056.0, + "16": 2842349056.0, + "17": 2842349056.0, + "18": 2842349056.0, + "19": 2842349056.0, + "20": 2842349056.0, + "21": 2842349056.0, + "22": 2842349056.0, + "23": 2842349056.0, + "24": 2842349056.0, + "25": 2842349056.0, + "26": 2842349056.0, + "27": 2842349056.0, + "28": 2842349056.0, + "29": 2842349056.0, + "30": 2842349056.0, + "31": 2842349056.0, + "32": 2842349056.0, + "33": 2842349056.0, + "34": 2842349056.0, + "35": 2842349056.0, + "36": 2842349056.0, + "37": 2842349056.0, + "38": 2842349056.0, + "39": 2842349056.0, + "40": 2842349056.0, + "41": 2842349056.0, + "42": 2842349056.0, + "43": 2842349056.0, + "44": 2842349056.0, + "45": 2842349056.0, + "46": 2842349056.0, + "47": 2842349056.0, + "48": 2842349056.0, + "49": 2842349056.0, + "50": 2842349056.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.38675, - "3": 0.07927, - "4": 0.0641, - "5": 0.06359, - "6": 0.06299, - "7": 0.06298, - "8": 0.0626, - "9": 0.06214, - "10": 0.06271, - "11": 0.06313, - "12": 0.06343, - "13": 0.06279, - "14": 0.06297, - "15": 0.06374, - "16": 0.06403, - "17": 0.06384, - "18": 0.064, - "19": 0.06239, - "20": 0.06381, - "21": 0.06262, - "22": 0.0633, - "23": 0.06345, - "24": 0.06201, - "25": 0.0634, - "26": 0.06353, - "27": 0.06324, - "28": 0.06249, - "29": 0.06236, - "30": 0.06263, - "31": 0.06341, - "32": 0.06385, - "33": 0.06297, - "34": 0.06348, - "35": 0.06265, - "36": 0.06278, - "37": 0.06324, - "38": 0.06353, - "39": 0.06332, - "40": 0.06394, - "41": 0.06402, - "42": 0.06356, - "43": 0.0639, - "44": 0.06452, - "45": 0.06334, - "46": 0.06329, - "47": 0.06325, - "48": 0.06343, - "49": 0.06358, - "50": 0.06333 + "2": 10.46661, + "3": 0.10862, + "4": 0.07855, + "5": 0.17395, + "6": 0.0601, + "7": 0.07016, + "8": 0.25646, + "9": 0.0599, + "10": 0.07586, + "11": 0.0782, + "12": 0.06037, + "13": 0.06445, + "14": 0.08794, + "15": 0.06099, + "16": 0.06121, + "17": 0.06107, + "18": 0.06329, + "19": 0.06007, + "20": 0.0631, + "21": 0.06203, + "22": 0.22063, + "23": 0.09969, + "24": 0.0609, + "25": 0.06373, + "26": 0.07271, + "27": 0.10238, + "28": 0.06408, + "29": 0.06171, + "30": 0.06243, + "31": 0.06024, + "32": 0.06465, + "33": 0.06232, + "34": 0.05988, + "35": 0.06234, + "36": 0.06591, + "37": 0.06261, + "38": 0.06162, + "39": 0.06209, + "40": 0.06037, + "41": 0.06221, + "42": 0.06136, + "43": 0.06264, + "44": 0.05972, + "45": 0.23878, + "46": 0.05897, + "47": 0.21809, + "48": 0.06156, + "49": 0.06198, + "50": 0.06272 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json index bc10a7b3abd..db3f8ab86c4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92679, - "2": 10.92501, - "3": 10.92521, - "4": 10.92422, - "5": 10.92547, - "6": 10.91033, - "7": 10.91228, - "8": 10.9229, - "9": 10.92921, - "10": 10.93096, - "11": 10.92107, + "1": 10.92677, + "2": 10.92498, + "3": 10.9252, + "4": 10.92429, + "5": 10.92541, + "6": 10.91034, + "7": 10.91232, + "8": 10.92285, + "9": 10.92916, + "10": 10.93091, + "11": 10.92106, "12": 10.9162, - "13": 10.91442, - "14": 10.91553, - "15": 10.91878, - "16": 10.90576, - "17": 10.91144, - "18": 10.89352, - "19": 10.9045, - "20": 10.88089, - "21": 10.86586, + "13": 10.91435, + "14": 10.91551, + "15": 10.91882, + "16": 10.90582, + "17": 10.91136, + "18": 10.89359, + "19": 10.90454, + "20": 10.8809, + "21": 10.86585, "22": 10.85398, - "23": 10.85622, - "24": 10.84644, - "25": 10.8478, - "26": 10.84328, + "23": 10.85608, + "24": 10.84649, + "25": 10.84777, + "26": 10.84327, "27": 10.83811, - "28": 10.76378, - "29": 10.75291, - "30": 10.74291, - "31": 10.71651, - "32": 10.7293, - "33": 10.70833, - "34": 10.67987, - "35": 10.67977, - "36": 10.67754, - "37": 10.64937, - "38": 10.62141, - "39": 10.58145, - "40": 10.5672, - "41": 10.5547, - "42": 10.52441, - "43": 10.5169, - "44": 10.49635, - "45": 10.50699, - "46": 10.46295, - "47": 10.464, - "48": 10.42072, - "49": 10.40804, - "50": 10.39052 + "28": 10.76387, + "29": 10.75296, + "30": 10.74298, + "31": 10.71659, + "32": 10.72937, + "33": 10.70832, + "34": 10.67995, + "35": 10.67987, + "36": 10.67766, + "37": 10.64938, + "38": 10.62144, + "39": 10.58151, + "40": 10.56728, + "41": 10.55472, + "42": 10.52448, + "43": 10.51695, + "44": 10.49638, + "45": 10.50711, + "46": 10.46296, + "47": 10.46401, + "48": 10.42075, + "49": 10.40821, + "50": 10.39062 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22961676.0, - "2": 22849608.0, - "3": 22708396.0, - "4": 22776906.0, - "5": 22783086.0, - "6": 22744204.0, - "7": 22870900.0, - "8": 22610356.0, - "9": 22758672.0, - "10": 22488532.0, - "11": 22754076.0, - "12": 22641504.0, - "13": 23321644.0, - "14": 22991400.0, - "15": 22717964.0, - "16": 22822160.0, - "17": 22928174.0, - "18": 22993726.0, - "19": 23083696.0, - "20": 22722568.0, - "21": 22916920.0, - "22": 22944496.0, - "23": 22627968.0, - "24": 22861256.0, - "25": 22635224.0, + "1": 22961644.0, + "2": 22849640.0, + "3": 22708490.0, + "4": 22776876.0, + "5": 22783124.0, + "6": 22744210.0, + "7": 22870980.0, + "8": 22610342.0, + "9": 22758686.0, + "10": 22488428.0, + "11": 22754040.0, + "12": 22641422.0, + "13": 23321638.0, + "14": 22991496.0, + "15": 22717886.0, + "16": 22822172.0, + "17": 22928028.0, + "18": 22993656.0, + "19": 23083736.0, + "20": 22722640.0, + "21": 22916940.0, + "22": 22944400.0, + "23": 22627992.0, + "24": 22861248.0, + "25": 22635202.0, "26": 23004420.0, - "27": 22800372.0, - "28": 22998250.0, - "29": 22977944.0, - "30": 22943312.0, - "31": 22906826.0, - "32": 22665452.0, - "33": 22740024.0, - "34": 23074180.0, - "35": 22749940.0, - "36": 22694928.0, - "37": 23102576.0, - "38": 22963904.0, - "39": 22983708.0, - "40": 22748514.0, - "41": 23064272.0, - "42": 22687828.0, - "43": 22986744.0, - "44": 22703500.0, - "45": 22845972.0, - "46": 22732688.0, - "47": 22848320.0, - "48": 22831952.0, - "49": 22887420.0, - "50": 22643594.0 + "27": 22800376.0, + "28": 22998380.0, + "29": 22977832.0, + "30": 22943328.0, + "31": 22906864.0, + "32": 22665466.0, + "33": 22740194.0, + "34": 23074260.0, + "35": 22749998.0, + "36": 22694964.0, + "37": 23102570.0, + "38": 22963848.0, + "39": 22983736.0, + "40": 22748524.0, + "41": 23064284.0, + "42": 22687892.0, + "43": 22986764.0, + "44": 22703484.0, + "45": 22845986.0, + "46": 22732752.0, + "47": 22848242.0, + "48": 22832138.0, + "49": 22887472.0, + "50": 22643654.0 } }, "mem-allocated-bytes": { @@ -179,8 +179,8 @@ "2": 2840251904.0, "3": 2840251904.0, "4": 2840251904.0, - "5": 2841169408.0, - "6": 2841169408.0, + "5": 2840251904.0, + "6": 2840251904.0, "7": 2841169408.0, "8": 2841169408.0, "9": 2841169408.0, @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.85847, - "3": 0.07317, - "4": 0.07284, - "5": 0.07195, - "6": 0.07422, - "7": 0.07283, - "8": 0.07174, - "9": 0.07266, - "10": 0.0717, - "11": 0.07102, - "12": 0.07181, - "13": 0.07157, - "14": 0.07022, - "15": 0.07232, - "16": 0.07228, - "17": 0.07172, - "18": 0.07174, - "19": 0.07099, - "20": 0.07215, - "21": 0.07285, - "22": 0.07143, - "23": 0.0719, - "24": 0.07066, - "25": 0.0709, - "26": 0.07287, - "27": 0.07095, - "28": 0.07074, - "29": 0.07198, - "30": 0.07139, - "31": 0.07124, - "32": 0.07122, - "33": 0.07196, - "34": 0.07084, - "35": 0.07457, - "36": 0.07092, - "37": 0.07103, - "38": 0.07077, - "39": 0.07044, - "40": 0.07129, - "41": 0.07181, - "42": 0.07137, - "43": 0.07124, - "44": 0.07128, - "45": 0.07014, - "46": 0.07083, - "47": 0.07142, - "48": 0.07092, - "49": 0.07082, - "50": 0.0712 + "2": 6.61232, + "3": 0.08231, + "4": 0.07993, + "5": 0.07756, + "6": 0.07313, + "7": 0.07605, + "8": 0.07418, + "9": 0.07253, + "10": 0.07549, + "11": 0.07126, + "12": 0.07033, + "13": 0.07049, + "14": 0.07169, + "15": 0.07155, + "16": 0.07295, + "17": 0.0707, + "18": 0.07149, + "19": 0.07367, + "20": 0.07166, + "21": 0.07151, + "22": 0.07103, + "23": 0.07212, + "24": 0.07161, + "25": 0.0715, + "26": 0.0724, + "27": 0.07187, + "28": 0.07053, + "29": 0.07341, + "30": 0.07235, + "31": 0.07168, + "32": 0.07223, + "33": 0.07165, + "34": 0.07244, + "35": 0.07323, + "36": 0.07144, + "37": 0.07143, + "38": 0.07526, + "39": 0.0718, + "40": 0.07199, + "41": 0.0717, + "42": 0.07185, + "43": 0.07202, + "44": 0.0713, + "45": 0.0715, + "46": 0.07292, + "47": 0.07078, + "48": 0.07223, + "49": 0.07252, + "50": 0.07157 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json index 8ce195c0ecd..5b5ee868896 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.90673, - "2": 10.91263, - "3": 10.91376, - "4": 10.90507, - "5": 10.90666, - "6": 10.9031, - "7": 10.9059, - "8": 10.91387, - "9": 10.90878, - "10": 10.9237, - "11": 10.90423, + "1": 10.90666, + "2": 10.91256, + "3": 10.91382, + "4": 10.90512, + "5": 10.90674, + "6": 10.90308, + "7": 10.90577, + "8": 10.91388, + "9": 10.90873, + "10": 10.92368, + "11": 10.9042, "12": 10.90022, - "13": 10.91206, - "14": 10.91487, - "15": 10.89545, - "16": 10.88862, - "17": 10.89879, - "18": 10.88901, + "13": 10.91207, + "14": 10.91489, + "15": 10.8954, + "16": 10.88861, + "17": 10.89883, + "18": 10.88906, "19": 10.89342, - "20": 10.85175, + "20": 10.85169, "21": 10.85495, - "22": 10.85415, - "23": 10.84295, - "24": 10.82607, - "25": 10.82944, - "26": 10.83505, - "27": 10.81605, - "28": 10.76187, - "29": 10.73276, - "30": 10.71482, - "31": 10.70796, - "32": 10.71541, - "33": 10.683, - "34": 10.6638, - "35": 10.66095, - "36": 10.64805, - "37": 10.62756, - "38": 10.60871, - "39": 10.55474, - "40": 10.54268, - "41": 10.52944, - "42": 10.51104, - "43": 10.50087, - "44": 10.46797, - "45": 10.47461, - "46": 10.4319, - "47": 10.42659, - "48": 10.39161, - "49": 10.3871, - "50": 10.36849 + "22": 10.85414, + "23": 10.84293, + "24": 10.8261, + "25": 10.82939, + "26": 10.83508, + "27": 10.8161, + "28": 10.76185, + "29": 10.73282, + "30": 10.7148, + "31": 10.70789, + "32": 10.71547, + "33": 10.68298, + "34": 10.66386, + "35": 10.66104, + "36": 10.64814, + "37": 10.62766, + "38": 10.60876, + "39": 10.55484, + "40": 10.54274, + "41": 10.52951, + "42": 10.51109, + "43": 10.50092, + "44": 10.46807, + "45": 10.47463, + "46": 10.43196, + "47": 10.4267, + "48": 10.39165, + "49": 10.38717, + "50": 10.36854 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22962016.0, - "2": 22848584.0, - "3": 22708364.0, - "4": 22777148.0, - "5": 22782800.0, - "6": 22745854.0, - "7": 22870548.0, - "8": 22610756.0, - "9": 22757820.0, - "10": 22489088.0, - "11": 22753460.0, - "12": 22640528.0, - "13": 23321632.0, - "14": 22990940.0, - "15": 22717336.0, - "16": 22821172.0, - "17": 22928714.0, - "18": 22994028.0, - "19": 23084446.0, - "20": 22722252.0, - "21": 22917710.0, - "22": 22945000.0, - "23": 22627552.0, - "24": 22861514.0, + "1": 22962108.0, + "2": 22848672.0, + "3": 22708304.0, + "4": 22777134.0, + "5": 22782820.0, + "6": 22745902.0, + "7": 22870540.0, + "8": 22610754.0, + "9": 22757862.0, + "10": 22489126.0, + "11": 22753440.0, + "12": 22640536.0, + "13": 23321504.0, + "14": 22990882.0, + "15": 22717276.0, + "16": 22821292.0, + "17": 22928660.0, + "18": 22994056.0, + "19": 23084428.0, + "20": 22722272.0, + "21": 22917716.0, + "22": 22944964.0, + "23": 22627636.0, + "24": 22861476.0, "25": 22635880.0, - "26": 23004104.0, - "27": 22801224.0, - "28": 22998068.0, - "29": 22978356.0, - "30": 22943064.0, - "31": 22906302.0, - "32": 22664320.0, - "33": 22739984.0, - "34": 23075192.0, - "35": 22749556.0, - "36": 22693736.0, - "37": 23103292.0, - "38": 22964034.0, - "39": 22983724.0, - "40": 22749088.0, - "41": 23063492.0, - "42": 22687832.0, - "43": 22986674.0, - "44": 22703486.0, - "45": 22846044.0, - "46": 22732872.0, - "47": 22848216.0, - "48": 22832092.0, - "49": 22887284.0, - "50": 22642954.0 + "26": 23004070.0, + "27": 22801236.0, + "28": 22998040.0, + "29": 22978500.0, + "30": 22943076.0, + "31": 22906274.0, + "32": 22664220.0, + "33": 22739872.0, + "34": 23075182.0, + "35": 22749520.0, + "36": 22693664.0, + "37": 23103284.0, + "38": 22963964.0, + "39": 22983794.0, + "40": 22749016.0, + "41": 23063564.0, + "42": 22687764.0, + "43": 22986694.0, + "44": 22703464.0, + "45": 22846068.0, + "46": 22732996.0, + "47": 22848258.0, + "48": 22832112.0, + "49": 22887248.0, + "50": 22642880.0 } }, "mem-allocated-bytes": { @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 2610027008.0, - "2": 2841171456.0, + "2": 2841169408.0, "3": 2841171456.0, "4": 2841171456.0, "5": 2841171456.0, - "6": 2841171456.0, - "7": 2841171456.0, - "8": 2841171456.0, - "9": 2841171456.0, - "10": 2841171456.0, - "11": 2841171456.0, - "12": 2841171456.0, - "13": 2841171456.0, - "14": 2841171456.0, - "15": 2841171456.0, - "16": 2841171456.0, - "17": 2841171456.0, - "18": 2841171456.0, - "19": 2841171456.0, - "20": 2841171456.0, - "21": 2841171456.0, - "22": 2841171456.0, - "23": 2841171456.0, - "24": 2841171456.0, - "25": 2841171456.0, - "26": 2841171456.0, - "27": 2841171456.0, - "28": 2841171456.0, - "29": 2841171456.0, - "30": 2841171456.0, - "31": 2841171456.0, - "32": 2841171456.0, - "33": 2841171456.0, - "34": 2841171456.0, - "35": 2841171456.0, - "36": 2841171456.0, - "37": 2841171456.0, - "38": 2841171456.0, - "39": 2842088960.0, - "40": 2842088960.0, - "41": 2842088960.0, - "42": 2842088960.0, - "43": 2842088960.0, - "44": 2842088960.0, - "45": 2842088960.0, - "46": 2842088960.0, - "47": 2842088960.0, - "48": 2842088960.0, - "49": 2842088960.0, - "50": 2842088960.0 + "6": 2841826816.0, + "7": 2841826816.0, + "8": 2841826816.0, + "9": 2841826816.0, + "10": 2841826816.0, + "11": 2841826816.0, + "12": 2841826816.0, + "13": 2841826816.0, + "14": 2841826816.0, + "15": 2841826816.0, + "16": 2841826816.0, + "17": 2841826816.0, + "18": 2841826816.0, + "19": 2841826816.0, + "20": 2841826816.0, + "21": 2841826816.0, + "22": 2841826816.0, + "23": 2841826816.0, + "24": 2841826816.0, + "25": 2841826816.0, + "26": 2841826816.0, + "27": 2841826816.0, + "28": 2841826816.0, + "29": 2841826816.0, + "30": 2841826816.0, + "31": 2841826816.0, + "32": 2841826816.0, + "33": 2841826816.0, + "34": 2841826816.0, + "35": 2841826816.0, + "36": 2841826816.0, + "37": 2841826816.0, + "38": 2841826816.0, + "39": 2841826816.0, + "40": 2841826816.0, + "41": 2841826816.0, + "42": 2841826816.0, + "43": 2841826816.0, + "44": 2841826816.0, + "45": 2841826816.0, + "46": 2841826816.0, + "47": 2841826816.0, + "48": 2841826816.0, + "49": 2841826816.0, + "50": 2841826816.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.9373, - "3": 1.87779, - "4": 1.77285, - "5": 1.60706, - "6": 1.73856, - "7": 1.71512, - "8": 2.3102, - "9": 2.34207, - "10": 1.712, - "11": 2.04949, - "12": 1.51573, - "13": 1.96266, - "14": 1.61171, - "15": 1.56691, - "16": 1.85963, - "17": 2.43498, - "18": 1.90961, - "19": 1.75855, - "20": 1.86545, - "21": 2.07982, - "22": 1.38767, - "23": 1.67867, - "24": 1.60979, - "25": 2.12194, - "26": 2.38425, - "27": 1.70776, - "28": 1.8058, - "29": 1.63302, - "30": 1.59172, - "31": 1.4531, - "32": 2.25714, - "33": 1.57842, - "34": 1.57638, - "35": 2.03622, - "36": 2.02296, - "37": 1.65123, - "38": 1.47971, - "39": 1.94619, - "40": 1.46247, - "41": 1.79716, - "42": 2.3247, - "43": 1.63639, - "44": 1.53309, - "45": 1.96158, - "46": 1.87269, - "47": 1.81212, - "48": 2.09298, - "49": 1.44854, - "50": 1.77975 + "2": 6.19369, + "3": 1.82843, + "4": 1.85517, + "5": 1.3476, + "6": 1.78611, + "7": 1.51574, + "8": 2.06166, + "9": 2.11972, + "10": 2.13461, + "11": 2.47455, + "12": 1.97125, + "13": 2.11098, + "14": 2.36274, + "15": 1.59411, + "16": 2.17388, + "17": 2.75022, + "18": 1.96556, + "19": 1.95354, + "20": 1.59697, + "21": 1.74663, + "22": 1.76111, + "23": 1.8886, + "24": 1.51118, + "25": 2.17387, + "26": 1.96545, + "27": 1.97459, + "28": 1.69958, + "29": 1.42079, + "30": 1.7769, + "31": 1.42843, + "32": 2.43183, + "33": 1.47292, + "34": 1.54108, + "35": 1.60591, + "36": 1.74558, + "37": 1.97066, + "38": 1.32487, + "39": 2.18776, + "40": 2.16596, + "41": 2.34461, + "42": 2.41474, + "43": 1.81404, + "44": 1.96086, + "45": 2.66946, + "46": 1.68703, + "47": 1.79827, + "48": 1.98854, + "49": 1.75285, + "50": 2.26667 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json index cac244346ab..2f3a187b5f9 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9566, - "2": 10.94874, - "3": 10.95794, - "4": 10.94237, - "5": 10.94996, - "6": 10.94647, - "7": 10.95213, - "8": 10.93674, - "9": 10.94654, - "10": 10.93654, - "11": 10.9358, - "12": 10.93463, - "13": 10.92753, - "14": 10.92179, + "1": 10.95667, + "2": 10.9487, + "3": 10.95793, + "4": 10.94236, + "5": 10.94998, + "6": 10.94641, + "7": 10.95207, + "8": 10.93678, + "9": 10.94649, + "10": 10.93656, + "11": 10.93583, + "12": 10.93462, + "13": 10.92754, + "14": 10.92174, "15": 10.90532, - "16": 10.88762, - "17": 10.90004, - "18": 10.8849, - "19": 10.88815, - "20": 10.79892, - "21": 10.79114, - "22": 10.78409, - "23": 10.77364, - "24": 10.73745, - "25": 10.74958, - "26": 10.72505, - "27": 10.69216, + "16": 10.88771, + "17": 10.90005, + "18": 10.88478, + "19": 10.88818, + "20": 10.79891, + "21": 10.79126, + "22": 10.7841, + "23": 10.77369, + "24": 10.73753, + "25": 10.74962, + "26": 10.72508, + "27": 10.6922, "28": 10.60873, - "29": 10.58333, - "30": 10.57497, - "31": 10.56473, - "32": 10.54813, + "29": 10.58337, + "30": 10.57495, + "31": 10.56469, + "32": 10.54825, "33": 10.51528, "34": 10.47329, - "35": 10.47428, - "36": 10.46262, - "37": 10.42021, - "38": 10.43007, - "39": 10.39407, - "40": 10.37593, - "41": 10.35955, - "42": 10.34467, - "43": 10.31064, - "44": 10.28611, - "45": 10.29465, - "46": 10.26351, - "47": 10.24628, - "48": 10.2054, - "49": 10.19472, - "50": 10.2, - "51": 10.20512, - "52": 10.15586, - "53": 10.15623, - "54": 10.12258, - "55": 10.09879, - "56": 10.12175, - "57": 10.10692, - "58": 10.11905, - "59": 10.0695, + "35": 10.47429, + "36": 10.46269, + "37": 10.42028, + "38": 10.43024, + "39": 10.39414, + "40": 10.376, + "41": 10.35958, + "42": 10.34471, + "43": 10.31069, + "44": 10.2861, + "45": 10.29467, + "46": 10.26353, + "47": 10.24633, + "48": 10.20551, + "49": 10.19474, + "50": 10.20006, + "51": 10.20525, + "52": 10.15591, + "53": 10.15633, + "54": 10.12257, + "55": 10.09891, + "56": 10.12177, + "57": 10.10695, + "58": 10.11914, + "59": 10.06955, "60": 10.08416, - "61": 10.03727, - "62": 10.01082, + "61": 10.03731, + "62": 10.01091, "63": 10.07761, - "64": 10.04079, - "65": 10.01332, - "66": 10.0369, - "67": 10.01278, - "68": 9.97739, - "69": 9.99946, - "70": 9.98, - "71": 10.00767, - "72": 9.98298, - "73": 9.97933, - "74": 9.96026, + "64": 10.04086, + "65": 10.01333, + "66": 10.03694, + "67": 10.01281, + "68": 9.97737, + "69": 9.99948, + "70": 9.98002, + "71": 10.00771, + "72": 9.98304, + "73": 9.97931, + "74": 9.9603, "75": 9.93508, - "76": 9.96763, - "77": 9.96093, - "78": 9.91032, - "79": 9.9179, + "76": 9.96768, + "77": 9.961, + "78": 9.91028, + "79": 9.91784, "80": 9.9381, - "81": 9.9632, - "82": 9.89645, - "83": 9.8601, - "84": 9.7882, - "85": 9.78784, - "86": 9.88494, - "87": 9.91253, + "81": 9.96325, + "82": 9.89649, + "83": 9.86012, + "84": 9.78819, + "85": 9.78793, + "86": 9.88503, + "87": 9.91256, "88": 9.88741, - "89": 9.81942, - "90": 9.81182, - "91": 9.83103, - "92": 9.81763, - "93": 9.74834, - "94": 9.83375, - "95": 9.82899, - "96": 9.80694, - "97": 9.74422, - "98": 9.78198, - "99": 9.82375, - "100": 9.71727 + "89": 9.8195, + "90": 9.81184, + "91": 9.83105, + "92": 9.81759, + "93": 9.74836, + "94": 9.83377, + "95": 9.82906, + "96": 9.80701, + "97": 9.74431, + "98": 9.78205, + "99": 9.82377, + "100": 9.7173 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1797.0, - "2": 1682.0, - "3": 1722.0, - "4": 1766.0, - "5": 1759.0, - "6": 1727.0, - "7": 1959.0, - "8": 1694.0, - "9": 1795.0, - "10": 1670.0, - "11": 1626.0, - "12": 1694.0, - "13": 1757.0, - "14": 1803.0, - "15": 1592.0, - "16": 1760.0, - "17": 1741.0, - "18": 1763.0, - "19": 1717.0, - "20": 1699.0, - "21": 1786.0, - "22": 1827.0, - "23": 1654.0, - "24": 1842.0, - "25": 1734.0, - "26": 1883.0, - "27": 1760.0, - "28": 1766.0, - "29": 1934.0, - "30": 1838.0, - "31": 1991.0, - "32": 1913.0, - "33": 1932.0, - "34": 1966.0, - "35": 2159.0, - "36": 2069.0, - "37": 2319.0, - "38": 2016.0, - "39": 2161.0, - "40": 2228.0, - "41": 2356.0, - "42": 1966.0, - "43": 2461.0, - "44": 2175.0, - "45": 2463.0, - "46": 2437.0, - "47": 2426.0, - "48": 2651.0, - "49": 2780.0, - "50": 2575.0, - "51": 2428.0, - "52": 2795.0, - "53": 2612.0, - "54": 2828.0, - "55": 2596.0, - "56": 2782.0, - "57": 2160.0, - "58": 3581.0, - "59": 2988.0, - "60": 3123.0, - "61": 2735.0, - "62": 3219.0, - "63": 3456.0, - "64": 3708.0, - "65": 2752.0, - "66": 3065.0, - "67": 3861.0, - "68": 3404.0, + "1": 1883.0, + "2": 1677.0, + "3": 1714.0, + "4": 1726.0, + "5": 1787.0, + "6": 1705.0, + "7": 1843.0, + "8": 1736.0, + "9": 1781.0, + "10": 1709.0, + "11": 1693.0, + "12": 1708.0, + "13": 1771.0, + "14": 1749.0, + "15": 1640.0, + "16": 1782.0, + "17": 1796.0, + "18": 1797.0, + "19": 1723.0, + "20": 1667.0, + "21": 1628.0, + "22": 1731.0, + "23": 1734.0, + "24": 1831.0, + "25": 1695.0, + "26": 1888.0, + "27": 1816.0, + "28": 1775.0, + "29": 1914.0, + "30": 1847.0, + "31": 2017.0, + "32": 1945.0, + "33": 2059.0, + "34": 2043.0, + "35": 2167.0, + "36": 1993.0, + "37": 2318.0, + "38": 1991.0, + "39": 2142.0, + "40": 2315.0, + "41": 2380.0, + "42": 2109.0, + "43": 2483.0, + "44": 2160.0, + "45": 2506.0, + "46": 2386.0, + "47": 2403.0, + "48": 2590.0, + "49": 2785.0, + "50": 2577.0, + "51": 2435.0, + "52": 2706.0, + "53": 2692.0, + "54": 2850.0, + "55": 2619.0, + "56": 2783.0, + "57": 2220.0, + "58": 3600.0, + "59": 2890.0, + "60": 3118.0, + "61": 2741.0, + "62": 3204.0, + "63": 3487.0, + "64": 3827.0, + "65": 2735.0, + "66": 3190.0, + "67": 3858.0, + "68": 3402.0, "69": 3085.0, - "70": 3362.0, - "71": 3223.0, - "72": 2975.0, - "73": 3385.0, - "74": 3364.0, - "75": 3225.0, - "76": 3189.0, - "77": 3824.0, - "78": 3369.0, - "79": 3372.0, - "80": 3195.0, - "81": 3651.0, - "82": 2919.0, - "83": 3152.0, - "84": 2978.0, - "85": 2829.0, - "86": 2963.0, - "87": 3066.0, - "88": 3098.0, - "89": 3191.0, - "90": 3834.0, - "91": 2907.0, - "92": 2825.0, - "93": 3082.0, - "94": 2914.0, - "95": 3110.0, - "96": 3334.0, - "97": 3445.0, - "98": 3382.0, - "99": 3278.0, - "100": 3303.0 + "70": 3333.0, + "71": 3350.0, + "72": 2879.0, + "73": 3422.0, + "74": 3338.0, + "75": 3137.0, + "76": 3199.0, + "77": 3893.0, + "78": 3446.0, + "79": 3458.0, + "80": 3120.0, + "81": 3629.0, + "82": 2880.0, + "83": 3207.0, + "84": 2985.0, + "85": 2760.0, + "86": 2824.0, + "87": 3071.0, + "88": 3056.0, + "89": 3177.0, + "90": 3927.0, + "91": 2873.0, + "92": 2752.0, + "93": 3095.0, + "94": 2906.0, + "95": 3119.0, + "96": 3200.0, + "97": 3479.0, + "98": 3482.0, + "99": 3312.0, + "100": 3339.0 } }, "mem-allocated-bytes": { @@ -375,56 +375,56 @@ "48": 1706868224.0, "49": 1706868224.0, "50": 1706868224.0, - "51": 1707915264.0, - "52": 1707915264.0, - "53": 1707915264.0, - "54": 1707915264.0, - "55": 1707915264.0, - "56": 1707915264.0, - "57": 1707915264.0, - "58": 1707915264.0, - "59": 1707915264.0, - "60": 1707915264.0, - "61": 1707915264.0, - "62": 1707915264.0, - "63": 1707915264.0, - "64": 1707915264.0, - "65": 1707915264.0, - "66": 1707915264.0, - "67": 1707915264.0, - "68": 1707915264.0, - "69": 1707915264.0, - "70": 1707915264.0, - "71": 1707915264.0, - "72": 1707915264.0, - "73": 1707915264.0, - "74": 1707915264.0, - "75": 1707915264.0, - "76": 1707915264.0, - "77": 1707915264.0, - "78": 1707915264.0, - "79": 1707915264.0, - "80": 1707915264.0, - "81": 1707915264.0, - "82": 1707915264.0, - "83": 1707915264.0, - "84": 1707915264.0, - "85": 1707915264.0, - "86": 1707915264.0, - "87": 1707915264.0, - "88": 1707915264.0, - "89": 1707915264.0, - "90": 1707915264.0, - "91": 1707915264.0, - "92": 1707915264.0, - "93": 1707915264.0, - "94": 1707915264.0, - "95": 1707915264.0, - "96": 1707915264.0, - "97": 1707915264.0, - "98": 1707915264.0, - "99": 1707915264.0, - "100": 1707915264.0 + "51": 1708441088.0, + "52": 1708441088.0, + "53": 1708441088.0, + "54": 1708441088.0, + "55": 1708441088.0, + "56": 1708441088.0, + "57": 1708441088.0, + "58": 1708441088.0, + "59": 1708441088.0, + "60": 1708441088.0, + "61": 1708441088.0, + "62": 1708441088.0, + "63": 1708441088.0, + "64": 1708441088.0, + "65": 1708441088.0, + "66": 1708441088.0, + "67": 1708441088.0, + "68": 1708441088.0, + "69": 1708441088.0, + "70": 1708441088.0, + "71": 1708441088.0, + "72": 1708441088.0, + "73": 1708441088.0, + "74": 1708441088.0, + "75": 1708441088.0, + "76": 1708441088.0, + "77": 1708441088.0, + "78": 1708441088.0, + "79": 1708441088.0, + "80": 1708441088.0, + "81": 1708441088.0, + "82": 1708441088.0, + "83": 1708441088.0, + "84": 1708441088.0, + "85": 1708441088.0, + "86": 1708441088.0, + "87": 1708441088.0, + "88": 1708441088.0, + "89": 1708441088.0, + "90": 1708441088.0, + "91": 1708441088.0, + "92": 1708441088.0, + "93": 1708441088.0, + "94": 1708441088.0, + "95": 1708441088.0, + "96": 1708441088.0, + "97": 1708441088.0, + "98": 1708441088.0, + "99": 1708441088.0, + "100": 1708441088.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.762, - "3": 0.18759, - "4": 0.17402, - "5": 0.17448, - "6": 0.17547, - "7": 0.17704, - "8": 0.17502, - "9": 0.17603, - "10": 0.17642, - "11": 0.1772, - "12": 0.17403, - "13": 0.17188, - "14": 0.17404, - "15": 0.17575, - "16": 0.17405, - "17": 0.17304, - "18": 0.17426, - "19": 0.17493, - "20": 0.17495, - "21": 0.17587, - "22": 0.1751, - "23": 0.17351, - "24": 0.17365, - "25": 0.17346, - "26": 0.1728, - "27": 0.17456, - "28": 0.17592, - "29": 0.17551, - "30": 0.17454, - "31": 0.17635, - "32": 0.17298, - "33": 0.17388, - "34": 0.17288, - "35": 0.17403, - "36": 0.1746, - "37": 0.17632, - "38": 0.17481, - "39": 0.1745, - "40": 0.17478, - "41": 0.17614, - "42": 0.17409, - "43": 0.17581, - "44": 0.17443, - "45": 0.17499, - "46": 0.17477, - "47": 0.1747, - "48": 0.17456, - "49": 0.17495, - "50": 0.17362, - "51": 0.23175, - "52": 0.22311, - "53": 0.24242, - "54": 0.19129, - "55": 0.17928, - "56": 0.17499, - "57": 0.17486, - "58": 0.17539, - "59": 0.17556, - "60": 0.17571, - "61": 0.17665, - "62": 0.17517, - "63": 0.17502, - "64": 0.17608, - "65": 0.17442, - "66": 0.17499, - "67": 0.17582, - "68": 0.17563, - "69": 0.17515, - "70": 0.17707, - "71": 0.17664, - "72": 0.17568, - "73": 0.17612, - "74": 0.17609, - "75": 0.17649, - "76": 0.17728, - "77": 0.17649, - "78": 0.17619, - "79": 0.17595, - "80": 0.17587, - "81": 0.17635, - "82": 0.17597, - "83": 0.17626, - "84": 0.17528, - "85": 0.17645, - "86": 0.17593, - "87": 0.17698, - "88": 0.17591, - "89": 0.17595, - "90": 0.17611, - "91": 0.17679, - "92": 0.17551, - "93": 0.17657, - "94": 0.17733, - "95": 0.1754, - "96": 0.17576, - "97": 0.17611, - "98": 0.17721, - "99": 0.17803, - "100": 0.19913 + "2": 7.64341, + "3": 0.19116, + "4": 0.15296, + "5": 0.15444, + "6": 0.15462, + "7": 0.15389, + "8": 0.15332, + "9": 0.15411, + "10": 0.15217, + "11": 0.1551, + "12": 0.15306, + "13": 0.15306, + "14": 0.15315, + "15": 0.15411, + "16": 0.15467, + "17": 0.15476, + "18": 0.15394, + "19": 0.15417, + "20": 0.15399, + "21": 0.15347, + "22": 0.15643, + "23": 0.1522, + "24": 0.15659, + "25": 0.15598, + "26": 0.15816, + "27": 0.15771, + "28": 0.1598, + "29": 0.16011, + "30": 0.16018, + "31": 0.16036, + "32": 0.15874, + "33": 0.15745, + "34": 0.15733, + "35": 0.1578, + "36": 0.15575, + "37": 0.1569, + "38": 0.15558, + "39": 0.15991, + "40": 0.16053, + "41": 0.15619, + "42": 0.15784, + "43": 0.15794, + "44": 0.15475, + "45": 0.1598, + "46": 0.15937, + "47": 0.15754, + "48": 0.15714, + "49": 0.15921, + "50": 0.15796, + "51": 0.25427, + "52": 0.21664, + "53": 0.15987, + "54": 0.16088, + "55": 0.15976, + "56": 0.15762, + "57": 0.16223, + "58": 0.15978, + "59": 0.16488, + "60": 0.1601, + "61": 0.1595, + "62": 0.15943, + "63": 0.16109, + "64": 0.15969, + "65": 0.15899, + "66": 0.16221, + "67": 0.15838, + "68": 0.16002, + "69": 0.16009, + "70": 0.1598, + "71": 0.15967, + "72": 0.15844, + "73": 0.15919, + "74": 0.15899, + "75": 0.15871, + "76": 0.16044, + "77": 0.15942, + "78": 0.16023, + "79": 0.15866, + "80": 0.15819, + "81": 0.15881, + "82": 0.15784, + "83": 0.16099, + "84": 0.15876, + "85": 0.16017, + "86": 0.15846, + "87": 0.15849, + "88": 0.16416, + "89": 0.15978, + "90": 0.15961, + "91": 0.1585, + "92": 0.1583, + "93": 0.15767, + "94": 0.15918, + "95": 0.16004, + "96": 0.15824, + "97": 0.16008, + "98": 0.15925, + "99": 0.15813, + "100": 0.15904 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json index 4c91171b41f..880b2b18161 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93132, - "2": 10.92661, - "3": 10.92492, - "4": 10.92081, - "5": 10.91909, - "6": 10.91583, - "7": 10.92398, - "8": 10.91668, - "9": 10.91511, - "10": 10.91953, + "2": 10.92668, + "3": 10.92486, + "4": 10.92085, + "5": 10.91905, + "6": 10.91578, + "7": 10.92394, + "8": 10.9167, + "9": 10.91517, + "10": 10.91951, "11": 10.91089, - "12": 10.90468, - "13": 10.89957, - "14": 10.9008, + "12": 10.90466, + "13": 10.89953, + "14": 10.90081, "15": 10.88316, - "16": 10.86332, - "17": 10.8821, - "18": 10.86576, - "19": 10.86883, - "20": 10.79012, - "21": 10.78745, - "22": 10.77483, - "23": 10.76341, - "24": 10.73372, - "25": 10.73326, - "26": 10.71699, - "27": 10.67915, - "28": 10.61678, - "29": 10.59025, - "30": 10.55953, - "31": 10.56388, - "32": 10.54876, - "33": 10.513, - "34": 10.48531, - "35": 10.48189, - "36": 10.46061, - "37": 10.43614, - "38": 10.43665, - "39": 10.39487, - "40": 10.3885, - "41": 10.36649, - "42": 10.34951, - "43": 10.32455, - "44": 10.30276, - "45": 10.30966, - "46": 10.27275, - "47": 10.25872, - "48": 10.2129, - "49": 10.21685, - "50": 10.21012, - "51": 10.21222, + "16": 10.86335, + "17": 10.88219, + "18": 10.86584, + "19": 10.86886, + "20": 10.79014, + "21": 10.78744, + "22": 10.77485, + "23": 10.76339, + "24": 10.73368, + "25": 10.73329, + "26": 10.71706, + "27": 10.67916, + "28": 10.61679, + "29": 10.59026, + "30": 10.55955, + "31": 10.56383, + "32": 10.54878, + "33": 10.51301, + "34": 10.48532, + "35": 10.48199, + "36": 10.46066, + "37": 10.43613, + "38": 10.43677, + "39": 10.3949, + "40": 10.38854, + "41": 10.36664, + "42": 10.34957, + "43": 10.32461, + "44": 10.30278, + "45": 10.3097, + "46": 10.2727, + "47": 10.25874, + "48": 10.213, + "49": 10.21691, + "50": 10.21015, + "51": 10.21226, "52": 10.16487, - "53": 10.17139, - "54": 10.13369, - "55": 10.10843, - "56": 10.13281, - "57": 10.11978, - "58": 10.12937, - "59": 10.07403, - "60": 10.09748, - "61": 10.05119, - "62": 10.01666, - "63": 10.0873, - "64": 10.03655, - "65": 10.0096, - "66": 10.04751, - "67": 10.02252, - "68": 9.98732, - "69": 10.00136, - "70": 9.98631, + "53": 10.17133, + "54": 10.13377, + "55": 10.10845, + "56": 10.13289, + "57": 10.11983, + "58": 10.12943, + "59": 10.07413, + "60": 10.09744, + "61": 10.0512, + "62": 10.01672, + "63": 10.08737, + "64": 10.03661, + "65": 10.00964, + "66": 10.04755, + "67": 10.02257, + "68": 9.98735, + "69": 10.00135, + "70": 9.98634, "71": 10.00761, - "72": 9.99372, - "73": 9.98419, - "74": 9.97133, - "75": 9.92977, - "76": 9.96826, - "77": 9.97256, - "78": 9.92117, - "79": 9.91774, - "80": 9.93559, - "81": 9.95704, + "72": 9.99378, + "73": 9.98423, + "74": 9.97138, + "75": 9.92982, + "76": 9.96831, + "77": 9.9726, + "78": 9.9212, + "79": 9.91778, + "80": 9.93564, + "81": 9.95713, "82": 9.8975, - "83": 9.86594, - "84": 9.80667, - "85": 9.78948, - "86": 9.89884, - "87": 9.9096, - "88": 9.89143, - "89": 9.8332, - "90": 9.82672, - "91": 9.83686, - "92": 9.82593, - "93": 9.76052, - "94": 9.83258, - "95": 9.82135, - "96": 9.80912, - "97": 9.75581, - "98": 9.78097, - "99": 9.82255, - "100": 9.71473 + "83": 9.86599, + "84": 9.8067, + "85": 9.78953, + "86": 9.89891, + "87": 9.90966, + "88": 9.89147, + "89": 9.83321, + "90": 9.82668, + "91": 9.83691, + "92": 9.82594, + "93": 9.76057, + "94": 9.8326, + "95": 9.82136, + "96": 9.80918, + "97": 9.7559, + "98": 9.78098, + "99": 9.82253, + "100": 9.71476 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1749.0, - "2": 1763.0, - "3": 1797.0, - "4": 1864.0, - "5": 1778.0, - "6": 1808.0, - "7": 1884.0, - "8": 1743.0, - "9": 1830.0, - "10": 1705.0, - "11": 1709.0, - "12": 1746.0, - "13": 1889.0, - "14": 1891.0, - "15": 1673.0, - "16": 1804.0, - "17": 1890.0, - "18": 1765.0, - "19": 1808.0, - "20": 1751.0, - "21": 1791.0, - "22": 1829.0, - "23": 1754.0, - "24": 1896.0, - "25": 1875.0, - "26": 1746.0, - "27": 1927.0, - "28": 1870.0, - "29": 1917.0, - "30": 1958.0, - "31": 2048.0, - "32": 1956.0, - "33": 1960.0, - "34": 2127.0, - "35": 2143.0, - "36": 2176.0, - "37": 2256.0, - "38": 2174.0, - "39": 2222.0, - "40": 2340.0, - "41": 2340.0, - "42": 2098.0, - "43": 2319.0, - "44": 2276.0, - "45": 2578.0, - "46": 2359.0, - "47": 2486.0, - "48": 2651.0, - "49": 2703.0, - "50": 2667.0, - "51": 2545.0, - "52": 2693.0, - "53": 2674.0, - "54": 2864.0, - "55": 2548.0, - "56": 2687.0, - "57": 2267.0, - "58": 3482.0, - "59": 2868.0, - "60": 2947.0, - "61": 2753.0, - "62": 3072.0, - "63": 3159.0, - "64": 3606.0, - "65": 2787.0, - "66": 2953.0, - "67": 3939.0, - "68": 3417.0, - "69": 2939.0, - "70": 3304.0, - "71": 3249.0, - "72": 3112.0, - "73": 3534.0, - "74": 3391.0, - "75": 3282.0, - "76": 3365.0, - "77": 3679.0, - "78": 3386.0, - "79": 3272.0, - "80": 3082.0, - "81": 3299.0, - "82": 3275.0, - "83": 3155.0, - "84": 3100.0, - "85": 2855.0, - "86": 3078.0, - "87": 3022.0, - "88": 3199.0, - "89": 3214.0, - "90": 4149.0, - "91": 2826.0, - "92": 3119.0, - "93": 3109.0, - "94": 3441.0, - "95": 3281.0, - "96": 3811.0, - "97": 3497.0, - "98": 3532.0, - "99": 3477.0, - "100": 3341.0 + "1": 1816.0, + "2": 1784.0, + "3": 1798.0, + "4": 1787.0, + "5": 1838.0, + "6": 1730.0, + "7": 2008.0, + "8": 1738.0, + "9": 1867.0, + "10": 1799.0, + "11": 1780.0, + "12": 1779.0, + "13": 1801.0, + "14": 1917.0, + "15": 1738.0, + "16": 1784.0, + "17": 1906.0, + "18": 1803.0, + "19": 1758.0, + "20": 1700.0, + "21": 1802.0, + "22": 1799.0, + "23": 1809.0, + "24": 1850.0, + "25": 1759.0, + "26": 1863.0, + "27": 1833.0, + "28": 1879.0, + "29": 2024.0, + "30": 1899.0, + "31": 1984.0, + "32": 2023.0, + "33": 2047.0, + "34": 2105.0, + "35": 2153.0, + "36": 2130.0, + "37": 2360.0, + "38": 2140.0, + "39": 2199.0, + "40": 2311.0, + "41": 2276.0, + "42": 2157.0, + "43": 2376.0, + "44": 2340.0, + "45": 2400.0, + "46": 2392.0, + "47": 2438.0, + "48": 2564.0, + "49": 2759.0, + "50": 2495.0, + "51": 2616.0, + "52": 2712.0, + "53": 2706.0, + "54": 2801.0, + "55": 2533.0, + "56": 2621.0, + "57": 2361.0, + "58": 3597.0, + "59": 2884.0, + "60": 2897.0, + "61": 2687.0, + "62": 3162.0, + "63": 3182.0, + "64": 3681.0, + "65": 2674.0, + "66": 3051.0, + "67": 3774.0, + "68": 3460.0, + "69": 2899.0, + "70": 3200.0, + "71": 3252.0, + "72": 3072.0, + "73": 3420.0, + "74": 3350.0, + "75": 3050.0, + "76": 3361.0, + "77": 3664.0, + "78": 3297.0, + "79": 3337.0, + "80": 3041.0, + "81": 3379.0, + "82": 3299.0, + "83": 3074.0, + "84": 3084.0, + "85": 2867.0, + "86": 3009.0, + "87": 2993.0, + "88": 3244.0, + "89": 3110.0, + "90": 4119.0, + "91": 3004.0, + "92": 3181.0, + "93": 3145.0, + "94": 3288.0, + "95": 3206.0, + "96": 3791.0, + "97": 3560.0, + "98": 3354.0, + "99": 3376.0, + "100": 3403.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.87058, - "3": 0.14701, - "4": 1.74329, - "5": 0.39339, - "6": 1.04265, - "7": 1.24208, - "8": 1.84418, - "9": 1.34809, - "10": 0.4529, - "11": 1.97844, - "12": 0.50437, - "13": 1.04936, - "14": 1.5431, - "15": 0.77848, - "16": 1.74381, - "17": 1.06464, - "18": 1.43904, - "19": 0.62659, - "20": 0.82054, - "21": 1.16783, - "22": 0.74085, - "23": 1.60571, - "24": 0.40254, - "25": 1.61209, - "26": 0.80727, - "27": 1.12032, - "28": 0.80779, - "29": 1.21616, - "30": 0.15192, - "31": 0.38017, - "32": 1.4174, - "33": 1.28283, - "34": 0.8669, - "35": 1.12703, - "36": 0.93849, - "37": 1.26457, - "38": 0.85287, - "39": 1.21165, - "40": 1.37614, - "41": 1.07684, - "42": 0.83559, - "43": 0.6735, - "44": 1.49902, - "45": 0.86528, - "46": 0.68184, - "47": 1.0296, - "48": 1.16521, - "49": 0.96442, - "50": 0.37884, - "51": 0.18367, - "52": 0.20666, - "53": 1.26983, - "54": 1.38124, - "55": 1.42823, - "56": 1.63453, - "57": 1.10568, - "58": 0.79131, - "59": 1.1396, - "60": 1.02703, - "61": 1.37674, - "62": 1.6992, - "63": 0.38795, - "64": 1.27667, - "65": 0.99906, - "66": 1.46019, - "67": 1.52888, - "68": 0.41083, - "69": 1.09554, - "70": 1.27494, - "71": 1.02772, - "72": 0.98292, - "73": 0.50204, - "74": 0.8382, - "75": 1.09369, - "76": 0.84162, - "77": 0.83643, - "78": 1.29648, - "79": 1.04829, - "80": 1.36637, - "81": 1.18688, - "82": 1.09685, - "83": 0.80535, - "84": 0.8312, - "85": 0.96307, - "86": 1.2134, - "87": 1.48783, - "88": 1.06343, - "89": 0.68606, - "90": 1.37786, - "91": 0.71266, - "92": 1.61009, - "93": 0.20605, - "94": 0.98568, - "95": 1.61806, - "96": 1.18797, - "97": 0.52742, - "98": 0.47026, - "99": 2.18459, - "100": 0.44562 + "2": 4.89908, + "3": 0.16123, + "4": 0.1608, + "5": 0.16072, + "6": 0.16358, + "7": 0.16059, + "8": 0.16128, + "9": 0.15845, + "10": 0.16018, + "11": 0.16167, + "12": 0.16341, + "13": 0.15854, + "14": 0.16107, + "15": 0.16171, + "16": 0.16128, + "17": 0.16339, + "18": 0.16085, + "19": 0.16133, + "20": 0.16359, + "21": 0.16299, + "22": 0.16303, + "23": 0.16599, + "24": 0.15967, + "25": 0.16095, + "26": 0.16029, + "27": 0.16021, + "28": 0.15875, + "29": 0.15962, + "30": 0.16168, + "31": 0.16095, + "32": 0.16124, + "33": 0.1593, + "34": 0.16006, + "35": 0.16061, + "36": 0.16102, + "37": 0.15988, + "38": 0.16076, + "39": 0.15908, + "40": 0.16022, + "41": 0.15847, + "42": 0.15982, + "43": 0.16061, + "44": 0.16268, + "45": 0.15953, + "46": 0.15812, + "47": 0.15888, + "48": 0.15856, + "49": 0.16104, + "50": 0.1599, + "51": 0.17928, + "52": 0.21523, + "53": 0.16547, + "54": 0.16098, + "55": 0.15955, + "56": 0.16458, + "57": 0.16727, + "58": 0.17723, + "59": 0.1641, + "60": 0.18417, + "61": 0.17383, + "62": 0.16702, + "63": 0.16139, + "64": 0.16137, + "65": 0.16072, + "66": 0.16423, + "67": 0.15984, + "68": 0.16127, + "69": 0.16196, + "70": 0.16, + "71": 0.15955, + "72": 0.15959, + "73": 0.1595, + "74": 0.15837, + "75": 0.16388, + "76": 0.16106, + "77": 0.16417, + "78": 0.16184, + "79": 0.16016, + "80": 0.16074, + "81": 0.16048, + "82": 0.15873, + "83": 0.16097, + "84": 0.16038, + "85": 0.16025, + "86": 0.15945, + "87": 0.16078, + "88": 0.16133, + "89": 0.16036, + "90": 0.16051, + "91": 0.16244, + "92": 0.15971, + "93": 0.15788, + "94": 0.15996, + "95": 0.15921, + "96": 0.1579, + "97": 0.15944, + "98": 0.15925, + "99": 0.15765, + "100": 0.15921 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json index b695b1b0405..cd92dca0481 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95267, - "2": 10.9487, + "1": 10.95268, + "2": 10.94868, "3": 10.95574, - "4": 10.94431, - "5": 10.94704, + "4": 10.94432, + "5": 10.94698, "6": 10.94549, - "7": 10.95237, - "8": 10.93777, - "9": 10.94948, - "10": 10.93746, - "11": 10.93723, - "12": 10.93039, - "13": 10.9262, - "14": 10.92334, - "15": 10.90308, - "16": 10.8917, - "17": 10.90057, - "18": 10.88579, - "19": 10.88639, - "20": 10.8026, - "21": 10.79321, - "22": 10.78322, - "23": 10.77587, - "24": 10.73896, - "25": 10.75417, - "26": 10.72426, - "27": 10.68676, - "28": 10.60946, - "29": 10.58265, - "30": 10.57715, - "31": 10.56009, - "32": 10.54731, - "33": 10.51567, - "34": 10.47312, - "35": 10.47454, - "36": 10.46305, - "37": 10.41894, - "38": 10.43058, - "39": 10.39286, - "40": 10.37587, - "41": 10.35957, - "42": 10.34601, - "43": 10.30948, - "44": 10.28618, - "45": 10.29673, - "46": 10.26334, - "47": 10.24621, - "48": 10.20438, - "49": 10.19586, - "50": 10.1985, - "51": 10.20538, - "52": 10.15528, - "53": 10.15622, - "54": 10.12185, - "55": 10.09957, - "56": 10.12109, - "57": 10.10642, - "58": 10.11963, - "59": 10.06902, - "60": 10.0833, - "61": 10.03794, - "62": 10.01121, - "63": 10.07814, - "64": 10.04133, - "65": 10.01344, - "66": 10.03614, - "67": 10.01253, - "68": 9.97682, - "69": 9.99821, + "7": 10.95231, + "8": 10.93786, + "9": 10.94945, + "10": 10.93747, + "11": 10.93726, + "12": 10.93046, + "13": 10.92612, + "14": 10.9233, + "15": 10.90304, + "16": 10.89173, + "17": 10.90056, + "18": 10.88581, + "19": 10.88646, + "20": 10.80252, + "21": 10.79332, + "22": 10.78326, + "23": 10.776, + "24": 10.73889, + "25": 10.7542, + "26": 10.72434, + "27": 10.68683, + "28": 10.60948, + "29": 10.58272, + "30": 10.57709, + "31": 10.56013, + "32": 10.54736, + "33": 10.5158, + "34": 10.47315, + "35": 10.4746, + "36": 10.46311, + "37": 10.41904, + "38": 10.4306, + "39": 10.39293, + "40": 10.37584, + "41": 10.3596, + "42": 10.34607, + "43": 10.30958, + "44": 10.28627, + "45": 10.29677, + "46": 10.26336, + "47": 10.24627, + "48": 10.20441, + "49": 10.19588, + "50": 10.19855, + "51": 10.20543, + "52": 10.15539, + "53": 10.15623, + "54": 10.12189, + "55": 10.09963, + "56": 10.12111, + "57": 10.10654, + "58": 10.11962, + "59": 10.06906, + "60": 10.08331, + "61": 10.03797, + "62": 10.01126, + "63": 10.07817, + "64": 10.04135, + "65": 10.01357, + "66": 10.03619, + "67": 10.01264, + "68": 9.97681, + "69": 9.99825, "70": 9.98055, - "71": 10.00729, - "72": 9.98249, - "73": 9.97937, - "74": 9.96109, - "75": 9.93579, - "76": 9.96744, + "71": 10.00732, + "72": 9.98252, + "73": 9.97936, + "74": 9.96117, + "75": 9.93586, + "76": 9.9674, "77": 9.96184, - "78": 9.90936, - "79": 9.9194, - "80": 9.9384, - "81": 9.96274, - "82": 9.89505, - "83": 9.85891, - "84": 9.7885, - "85": 9.7864, - "86": 9.88552, - "87": 9.91121, - "88": 9.88699, - "89": 9.8184, - "90": 9.81194, - "91": 9.83103, - "92": 9.81784, - "93": 9.74871, - "94": 9.83439, - "95": 9.82929, - "96": 9.80802, - "97": 9.74284, - "98": 9.78138, - "99": 9.82283, - "100": 9.71499 + "78": 9.90945, + "79": 9.91948, + "80": 9.93845, + "81": 9.96282, + "82": 9.89511, + "83": 9.85893, + "84": 9.78855, + "85": 9.78647, + "86": 9.8855, + "87": 9.91123, + "88": 9.88706, + "89": 9.81846, + "90": 9.81195, + "91": 9.8311, + "92": 9.81789, + "93": 9.74874, + "94": 9.8344, + "95": 9.82931, + "96": 9.80803, + "97": 9.74294, + "98": 9.78146, + "99": 9.82289, + "100": 9.71501 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1769.0, - "2": 1730.0, - "3": 1728.0, - "4": 1612.0, + "1": 1725.0, + "2": 1782.0, + "3": 1740.0, + "4": 1671.0, "5": 1692.0, - "6": 1719.0, - "7": 1872.0, - "8": 1657.0, - "9": 1770.0, - "10": 1777.0, - "11": 1697.0, - "12": 1721.0, - "13": 1783.0, - "14": 1784.0, - "15": 1583.0, - "16": 1711.0, - "17": 1743.0, - "18": 1766.0, - "19": 1755.0, - "20": 1676.0, - "21": 1838.0, - "22": 1726.0, - "23": 1650.0, - "24": 1801.0, - "25": 1822.0, - "26": 1818.0, - "27": 1857.0, - "28": 1830.0, - "29": 2024.0, - "30": 1772.0, - "31": 2004.0, - "32": 1984.0, - "33": 1939.0, - "34": 2014.0, - "35": 2190.0, - "36": 2029.0, - "37": 2171.0, - "38": 2076.0, - "39": 2307.0, - "40": 2283.0, - "41": 2352.0, - "42": 2038.0, - "43": 2462.0, - "44": 2201.0, - "45": 2496.0, - "46": 2332.0, - "47": 2431.0, - "48": 2657.0, - "49": 2706.0, - "50": 2523.0, - "51": 2422.0, - "52": 2630.0, - "53": 2633.0, - "54": 2831.0, - "55": 2584.0, - "56": 2781.0, - "57": 2244.0, - "58": 3551.0, - "59": 3006.0, - "60": 2928.0, - "61": 2894.0, - "62": 3200.0, - "63": 3409.0, - "64": 3588.0, - "65": 2745.0, - "66": 3199.0, - "67": 3933.0, - "68": 3489.0, - "69": 3109.0, - "70": 3355.0, - "71": 3067.0, - "72": 2965.0, - "73": 3434.0, - "74": 3343.0, - "75": 3203.0, - "76": 3315.0, - "77": 3776.0, - "78": 3323.0, - "79": 3260.0, - "80": 3268.0, - "81": 3533.0, - "82": 2993.0, - "83": 3035.0, - "84": 2907.0, - "85": 2822.0, - "86": 2891.0, - "87": 2942.0, - "88": 3135.0, - "89": 3215.0, - "90": 3838.0, - "91": 2968.0, - "92": 2759.0, - "93": 3024.0, - "94": 2973.0, - "95": 3242.0, - "96": 3295.0, - "97": 3308.0, - "98": 3192.0, - "99": 3267.0, - "100": 3362.0 + "6": 1676.0, + "7": 1929.0, + "8": 1720.0, + "9": 1785.0, + "10": 1742.0, + "11": 1670.0, + "12": 1737.0, + "13": 1795.0, + "14": 1918.0, + "15": 1547.0, + "16": 1719.0, + "17": 1865.0, + "18": 1875.0, + "19": 1679.0, + "20": 1720.0, + "21": 1731.0, + "22": 1688.0, + "23": 1661.0, + "24": 1764.0, + "25": 1707.0, + "26": 1885.0, + "27": 1793.0, + "28": 1739.0, + "29": 2016.0, + "30": 1881.0, + "31": 2021.0, + "32": 1964.0, + "33": 1920.0, + "34": 2041.0, + "35": 2165.0, + "36": 1937.0, + "37": 2235.0, + "38": 2103.0, + "39": 2236.0, + "40": 2307.0, + "41": 2386.0, + "42": 1985.0, + "43": 2454.0, + "44": 2209.0, + "45": 2458.0, + "46": 2418.0, + "47": 2407.0, + "48": 2631.0, + "49": 2875.0, + "50": 2608.0, + "51": 2459.0, + "52": 2730.0, + "53": 2675.0, + "54": 2815.0, + "55": 2609.0, + "56": 2717.0, + "57": 2284.0, + "58": 3631.0, + "59": 2852.0, + "60": 3061.0, + "61": 2899.0, + "62": 3199.0, + "63": 3452.0, + "64": 3583.0, + "65": 2865.0, + "66": 3228.0, + "67": 4159.0, + "68": 3369.0, + "69": 3139.0, + "70": 3416.0, + "71": 3141.0, + "72": 2939.0, + "73": 3516.0, + "74": 3368.0, + "75": 3165.0, + "76": 3242.0, + "77": 3852.0, + "78": 3411.0, + "79": 3341.0, + "80": 3118.0, + "81": 3689.0, + "82": 2867.0, + "83": 3164.0, + "84": 2952.0, + "85": 2814.0, + "86": 2865.0, + "87": 2967.0, + "88": 3115.0, + "89": 3211.0, + "90": 3736.0, + "91": 2846.0, + "92": 2832.0, + "93": 3031.0, + "94": 2966.0, + "95": 3381.0, + "96": 3246.0, + "97": 3364.0, + "98": 3195.0, + "99": 3301.0, + "100": 3320.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.79237, - "3": 1.60566, - "4": 1.20579, - "5": 0.99309, - "6": 1.33557, - "7": 1.39425, - "8": 1.47423, - "9": 1.60284, - "10": 1.12246, - "11": 1.15703, - "12": 1.16144, - "13": 1.32006, - "14": 1.47244, - "15": 0.9489, - "16": 1.11446, - "17": 1.47698, - "18": 1.25144, - "19": 1.27479, - "20": 1.27309, - "21": 1.29416, - "22": 1.21734, - "23": 1.28426, - "24": 1.8034, - "25": 0.76424, - "26": 1.08554, - "27": 1.0629, - "28": 1.54008, - "29": 0.95585, - "30": 0.88659, - "31": 0.98651, - "32": 1.67568, - "33": 1.13253, - "34": 0.45273, - "35": 1.487, - "36": 1.22764, - "37": 1.10544, - "38": 0.77458, - "39": 1.48211, - "40": 1.77497, - "41": 1.38665, - "42": 1.43913, - "43": 1.09094, - "44": 1.5532, - "45": 1.32776, - "46": 0.9677, - "47": 1.28294, - "48": 1.07283, - "49": 0.91723, - "50": 1.24133, - "51": 0.39892, - "52": 1.19418, - "53": 1.33471, - "54": 1.49197, - "55": 1.32517, - "56": 1.14412, - "57": 0.87043, - "58": 0.91378, - "59": 1.19857, - "60": 1.00932, - "61": 1.10432, - "62": 1.69769, - "63": 1.16786, - "64": 1.28708, - "65": 0.7877, - "66": 1.46358, - "67": 1.3682, - "68": 1.01245, - "69": 1.45405, - "70": 0.98541, - "71": 1.12885, - "72": 1.09485, - "73": 0.96702, - "74": 0.7814, - "75": 0.92313, - "76": 1.42561, - "77": 1.31904, - "78": 1.36731, - "79": 1.95785, - "80": 1.35427, - "81": 1.11772, - "82": 1.63776, - "83": 1.3614, - "84": 1.02251, - "85": 0.80518, - "86": 1.04223, - "87": 1.94232, - "88": 1.13576, - "89": 2.26714, - "90": 1.33072, - "91": 1.53862, - "92": 1.54225, - "93": 1.26952, - "94": 1.67887, - "95": 0.9208, - "96": 1.14527, - "97": 1.08586, - "98": 0.92, - "99": 1.74351, - "100": 1.20956 + "2": 4.50771, + "3": 1.36757, + "4": 1.06242, + "5": 0.77211, + "6": 1.21926, + "7": 1.1421, + "8": 1.38515, + "9": 1.39868, + "10": 0.91618, + "11": 1.04838, + "12": 0.99689, + "13": 1.16732, + "14": 1.27374, + "15": 0.83108, + "16": 0.90044, + "17": 1.21952, + "18": 1.08607, + "19": 1.0499, + "20": 1.03264, + "21": 1.02708, + "22": 0.99929, + "23": 1.13064, + "24": 1.58497, + "25": 0.76057, + "26": 0.76923, + "27": 0.96547, + "28": 1.12726, + "29": 0.91422, + "30": 0.92986, + "31": 0.78544, + "32": 1.75487, + "33": 1.07724, + "34": 0.38483, + "35": 1.29843, + "36": 1.31954, + "37": 0.93968, + "38": 0.73738, + "39": 0.97599, + "40": 1.37942, + "41": 0.99875, + "42": 1.13505, + "43": 0.7573, + "44": 1.28748, + "45": 1.13954, + "46": 0.74884, + "47": 1.13122, + "48": 0.93622, + "49": 0.92536, + "50": 1.19527, + "51": 0.33528, + "52": 1.01767, + "53": 1.17067, + "54": 1.28243, + "55": 1.21179, + "56": 0.98217, + "57": 0.85014, + "58": 0.84985, + "59": 1.01073, + "60": 0.89359, + "61": 0.86023, + "62": 1.46506, + "63": 0.96004, + "64": 1.00184, + "65": 0.67292, + "66": 1.18176, + "67": 1.20206, + "68": 0.87583, + "69": 1.3283, + "70": 0.92458, + "71": 1.07428, + "72": 1.09099, + "73": 0.95736, + "74": 0.73377, + "75": 0.80013, + "76": 1.30949, + "77": 1.20279, + "78": 1.19816, + "79": 1.55665, + "80": 1.15334, + "81": 0.99328, + "82": 1.36233, + "83": 1.22678, + "84": 1.11922, + "85": 0.73439, + "86": 1.02536, + "87": 1.67613, + "88": 1.01063, + "89": 1.03166, + "90": 1.09803, + "91": 0.92073, + "92": 1.4709, + "93": 1.01915, + "94": 1.17512, + "95": 1.05371, + "96": 1.00387, + "97": 0.95996, + "98": 1.20141, + "99": 1.63641, + "100": 1.06399 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json index 6a27a0bdc9f..ab6a4536da7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json @@ -6,53 +6,53 @@ "values": { "1": 10.9228, "2": 10.92259, - "3": 10.91129, - "4": 10.91262, - "5": 10.91229, - "6": 10.91115, - "7": 10.90761, - "8": 10.90486, - "9": 10.90445, - "10": 10.90371, - "11": 10.89156, - "12": 10.89897, - "13": 10.89488, + "3": 10.91131, + "4": 10.91261, + "5": 10.91231, + "6": 10.91112, + "7": 10.90763, + "8": 10.90489, + "9": 10.90448, + "10": 10.90372, + "11": 10.89157, + "12": 10.89898, + "13": 10.89492, "14": 10.88107, - "15": 10.85401, + "15": 10.85396, "16": 10.84622, - "17": 10.84596, + "17": 10.84592, "18": 10.83436, - "19": 10.83439, - "20": 10.73158, - "21": 10.71288, - "22": 10.71183, - "23": 10.70052, - "24": 10.66577, - "25": 10.66681, - "26": 10.65427, - "27": 10.59994, - "28": 10.53048, - "29": 10.50772, - "30": 10.49516, - "31": 10.47433, - "32": 10.4633, - "33": 10.42698, + "19": 10.83435, + "20": 10.73163, + "21": 10.71287, + "22": 10.71185, + "23": 10.70053, + "24": 10.66578, + "25": 10.66683, + "26": 10.65428, + "27": 10.59989, + "28": 10.53051, + "29": 10.50775, + "30": 10.49518, + "31": 10.47431, + "32": 10.46336, + "33": 10.42695, "34": 10.39676, - "35": 10.39597, - "36": 10.38022, - "37": 10.33527, - "38": 10.34443, - "39": 10.30848, - "40": 10.30596, - "41": 10.27175, - "42": 10.25554, - "43": 10.22764, - "44": 10.20321, - "45": 10.22047, - "46": 10.18449, - "47": 10.16912, - "48": 10.11941, - "49": 10.1263, + "35": 10.39601, + "36": 10.38026, + "37": 10.33528, + "38": 10.34442, + "39": 10.30845, + "40": 10.30599, + "41": 10.27178, + "42": 10.25552, + "43": 10.22763, + "44": 10.20322, + "45": 10.2205, + "46": 10.18447, + "47": 10.16913, + "48": 10.11938, + "49": 10.12631, "50": 10.13123 } }, @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 618.0, - "2": 609.0, - "3": 605.0, - "4": 631.0, - "5": 619.0, - "6": 616.0, - "7": 696.0, - "8": 613.0, - "9": 624.0, - "10": 661.0, - "11": 605.0, - "12": 597.0, - "13": 621.0, - "14": 662.0, - "15": 571.0, - "16": 614.0, - "17": 653.0, - "18": 629.0, - "19": 620.0, - "20": 639.0, - "21": 679.0, - "22": 603.0, - "23": 618.0, - "24": 650.0, - "25": 624.0, - "26": 663.0, - "27": 679.0, - "28": 728.0, - "29": 671.0, - "30": 717.0, - "31": 731.0, - "32": 748.0, - "33": 757.0, - "34": 726.0, - "35": 759.0, - "36": 758.0, - "37": 770.0, - "38": 819.0, - "39": 886.0, - "40": 840.0, - "41": 936.0, - "42": 784.0, - "43": 887.0, - "44": 849.0, - "45": 939.0, - "46": 925.0, - "47": 929.0, - "48": 948.0, - "49": 1027.0, - "50": 1050.0 + "1": 640.0, + "2": 605.0, + "3": 603.0, + "4": 605.0, + "5": 606.0, + "6": 585.0, + "7": 697.0, + "8": 602.0, + "9": 630.0, + "10": 624.0, + "11": 678.0, + "12": 583.0, + "13": 620.0, + "14": 626.0, + "15": 605.0, + "16": 598.0, + "17": 634.0, + "18": 601.0, + "19": 626.0, + "20": 565.0, + "21": 683.0, + "22": 619.0, + "23": 648.0, + "24": 699.0, + "25": 628.0, + "26": 657.0, + "27": 653.0, + "28": 642.0, + "29": 760.0, + "30": 705.0, + "31": 753.0, + "32": 740.0, + "33": 718.0, + "34": 766.0, + "35": 809.0, + "36": 722.0, + "37": 765.0, + "38": 782.0, + "39": 838.0, + "40": 915.0, + "41": 918.0, + "42": 739.0, + "43": 847.0, + "44": 864.0, + "45": 968.0, + "46": 940.0, + "47": 893.0, + "48": 1004.0, + "49": 1015.0, + "50": 1028.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 460882432.0, - "2": 460882432.0, - "3": 460882432.0, - "4": 460882432.0, - "5": 460882432.0, - "6": 460882432.0, - "7": 460882432.0, - "8": 460882432.0, - "9": 460882432.0, - "10": 460882432.0, - "11": 460882432.0, - "12": 460882432.0, - "13": 460882432.0, - "14": 460882432.0, - "15": 460882432.0, - "16": 460882432.0, - "17": 460882432.0, - "18": 460882432.0, - "19": 460882432.0, - "20": 460882432.0, - "21": 460882432.0, - "22": 460882432.0, - "23": 460882432.0, - "24": 460882432.0, - "25": 460882432.0, - "26": 460882432.0, - "27": 460882432.0, - "28": 460882432.0, - "29": 460882432.0, - "30": 460882432.0, - "31": 460882432.0, - "32": 460882432.0, - "33": 460882432.0, - "34": 460882432.0, - "35": 460882432.0, - "36": 460882432.0, - "37": 460882432.0, - "38": 460882432.0, - "39": 460882432.0, - "40": 460882432.0, - "41": 460882432.0, - "42": 460882432.0, - "43": 460882432.0, - "44": 460882432.0, - "45": 460882432.0, - "46": 460882432.0, - "47": 460882432.0, - "48": 460882432.0, - "49": 460882432.0, - "50": 460882432.0 + "1": 512262656.0, + "2": 512262656.0, + "3": 512262656.0, + "4": 512262656.0, + "5": 512262656.0, + "6": 512262656.0, + "7": 512262656.0, + "8": 512262656.0, + "9": 512262656.0, + "10": 512262656.0, + "11": 512262656.0, + "12": 512262656.0, + "13": 512262656.0, + "14": 512262656.0, + "15": 512262656.0, + "16": 512262656.0, + "17": 512262656.0, + "18": 512262656.0, + "19": 512262656.0, + "20": 512262656.0, + "21": 512262656.0, + "22": 512262656.0, + "23": 512262656.0, + "24": 512262656.0, + "25": 512262656.0, + "26": 512262656.0, + "27": 512262656.0, + "28": 512262656.0, + "29": 512262656.0, + "30": 512262656.0, + "31": 512262656.0, + "32": 512262656.0, + "33": 512262656.0, + "34": 512262656.0, + "35": 512262656.0, + "36": 512262656.0, + "37": 512262656.0, + "38": 512262656.0, + "39": 512262656.0, + "40": 512262656.0, + "41": 512262656.0, + "42": 512262656.0, + "43": 512262656.0, + "44": 512262656.0, + "45": 512262656.0, + "46": 512262656.0, + "47": 512262656.0, + "48": 512262656.0, + "49": 512262656.0, + "50": 512262656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 707731456.0, - "2": 885443072.0, - "3": 885443072.0, - "4": 885443072.0, - "5": 885443072.0, - "6": 885445120.0, - "7": 885445120.0, - "8": 885445120.0, - "9": 885445120.0, - "10": 885445120.0, - "11": 885445120.0, - "12": 885445120.0, - "13": 885445120.0, - "14": 885445120.0, - "15": 885445120.0, - "16": 885445120.0, - "17": 885445120.0, - "18": 885445120.0, - "19": 885445120.0, - "20": 885445120.0, - "21": 885445120.0, - "22": 885445120.0, - "23": 885445120.0, - "24": 885445120.0, - "25": 885445120.0, - "26": 885445120.0, - "27": 885445632.0, - "28": 885445632.0, - "29": 885445632.0, - "30": 885445632.0, - "31": 885445632.0, - "32": 885445632.0, - "33": 885445632.0, - "34": 885445632.0, - "35": 885445632.0, - "36": 885445632.0, - "37": 885445632.0, - "38": 885445632.0, - "39": 885445632.0, - "40": 885445632.0, - "41": 885445632.0, - "42": 885445632.0, - "43": 885445632.0, - "44": 885445632.0, - "45": 885445632.0, - "46": 885445632.0, - "47": 885445632.0, - "48": 885445632.0, - "49": 885445632.0, - "50": 885446144.0 + "1": 755704832.0, + "2": 936825856.0, + "3": 936825856.0, + "4": 936825856.0, + "5": 936825856.0, + "6": 936825856.0, + "7": 936825856.0, + "8": 936825856.0, + "9": 936825856.0, + "10": 936825856.0, + "11": 936825856.0, + "12": 936826368.0, + "13": 936826368.0, + "14": 936826368.0, + "15": 936826368.0, + "16": 936826368.0, + "17": 936826368.0, + "18": 936826368.0, + "19": 936826368.0, + "20": 936826368.0, + "21": 936826368.0, + "22": 936826368.0, + "23": 936826368.0, + "24": 936826368.0, + "25": 936826368.0, + "26": 936826368.0, + "27": 936826368.0, + "28": 936826368.0, + "29": 936826368.0, + "30": 936826368.0, + "31": 936826368.0, + "32": 936826368.0, + "33": 936826368.0, + "34": 936826368.0, + "35": 936826368.0, + "36": 936826368.0, + "37": 936826368.0, + "38": 936826368.0, + "39": 936826368.0, + "40": 936826368.0, + "41": 936826368.0, + "42": 936826368.0, + "43": 936826368.0, + "44": 936826368.0, + "45": 936826368.0, + "46": 936826368.0, + "47": 936826368.0, + "48": 936826368.0, + "49": 936826368.0, + "50": 936826368.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.59705, - "3": 0.46072, - "4": 0.45115, - "5": 0.45105, - "6": 0.45353, - "7": 0.45074, - "8": 0.45354, - "9": 0.45337, - "10": 0.45297, - "11": 0.45356, - "12": 0.45389, - "13": 0.45457, - "14": 0.45381, - "15": 0.45574, - "16": 0.45244, - "17": 0.45438, - "18": 0.45325, - "19": 0.44745, - "20": 0.44671, - "21": 0.44998, - "22": 0.45016, - "23": 0.44811, - "24": 0.4494, - "25": 0.44781, - "26": 0.44723, - "27": 0.44674, - "28": 0.4509, - "29": 0.44994, - "30": 0.44983, - "31": 0.45232, - "32": 0.4516, - "33": 0.45277, - "34": 0.44951, - "35": 0.46001, - "36": 0.45247, - "37": 0.45799, - "38": 0.45878, - "39": 0.45522, - "40": 0.44849, - "41": 0.45094, - "42": 0.45256, - "43": 0.45059, - "44": 0.45206, - "45": 0.44872, - "46": 0.44621, - "47": 0.44813, - "48": 0.45292, - "49": 0.45077, - "50": 0.45584 + "2": 11.43327, + "3": 0.47778, + "4": 0.42271, + "5": 0.42025, + "6": 0.43937, + "7": 0.41941, + "8": 0.41554, + "9": 0.41787, + "10": 0.41759, + "11": 0.41602, + "12": 0.41961, + "13": 0.42625, + "14": 0.42559, + "15": 0.42369, + "16": 0.42176, + "17": 0.41878, + "18": 0.41848, + "19": 0.42073, + "20": 0.41976, + "21": 0.41612, + "22": 0.41737, + "23": 0.4185, + "24": 0.4245, + "25": 0.42339, + "26": 0.42416, + "27": 0.4179, + "28": 0.41416, + "29": 0.41467, + "30": 0.41453, + "31": 0.41657, + "32": 0.4151, + "33": 0.41781, + "34": 0.4148, + "35": 0.42058, + "36": 0.41403, + "37": 0.41692, + "38": 0.41888, + "39": 0.41771, + "40": 0.41753, + "41": 0.42052, + "42": 0.4311, + "43": 0.42164, + "44": 0.4167, + "45": 0.41919, + "46": 0.41337, + "47": 0.425, + "48": 0.42356, + "49": 0.42178, + "50": 0.42379 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json index b327af5c817..92a53aefc66 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 637582848.0, - "2": 637582848.0, - "3": 637582848.0, - "4": 637582848.0, - "5": 637582848.0, - "6": 637582848.0, - "7": 637582848.0, - "8": 637582848.0, - "9": 637582848.0, - "10": 637582848.0, - "11": 637582848.0, - "12": 637582848.0, - "13": 637582848.0, - "14": 637582848.0, - "15": 637582848.0, - "16": 637582848.0, - "17": 637582848.0, - "18": 637582848.0, - "19": 637582848.0, - "20": 637582848.0, - "21": 637582848.0, - "22": 637582848.0, - "23": 637582848.0, - "24": 637582848.0, - "25": 637582848.0, - "26": 637582848.0, - "27": 637582848.0, - "28": 637582848.0, - "29": 637582848.0, - "30": 637582848.0, - "31": 637582848.0, - "32": 637582848.0, - "33": 637582848.0, - "34": 637582848.0, - "35": 637582848.0, - "36": 637582848.0, - "37": 637582848.0, - "38": 637582848.0, - "39": 637582848.0, - "40": 637582848.0, - "41": 637582848.0, - "42": 637582848.0, - "43": 637582848.0, - "44": 637582848.0, - "45": 637582848.0, - "46": 637582848.0, - "47": 637582848.0, - "48": 637582848.0, - "49": 637582848.0, - "50": 637582848.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 910503936.0, - "2": 1172074496.0, - "3": 1172075008.0, - "4": 1172075008.0, - "5": 1172075008.0, - "6": 1176268800.0, - "7": 1176268800.0, - "8": 1176268800.0, - "9": 1176268800.0, - "10": 1176268800.0, - "11": 1176268800.0, - "12": 1176268800.0, - "13": 1176268800.0, - "14": 1177316352.0, - "15": 1177316352.0, - "16": 1177316352.0, - "17": 1177316352.0, - "18": 1177316352.0, - "19": 1177316352.0, - "20": 1177316352.0, - "21": 1177316352.0, - "22": 1177316352.0, - "23": 1177316352.0, - "24": 1177316352.0, - "25": 1177316352.0, - "26": 1177316352.0, - "27": 1177316352.0, - "28": 1177316352.0, - "29": 1177316352.0, - "30": 1177316352.0, - "31": 1177316352.0, - "32": 1177316352.0, - "33": 1177316352.0, - "34": 1177316352.0, - "35": 1177316352.0, - "36": 1177316352.0, - "37": 1177316352.0, - "38": 1177316352.0, - "39": 1177316352.0, - "40": 1177316352.0, - "41": 1177316352.0, - "42": 1177316352.0, - "43": 1177316352.0, - "44": 1177316352.0, - "45": 1177316352.0, - "46": 1177316352.0, - "47": 1177316352.0, - "48": 1177316352.0, - "49": 1177316352.0, - "50": 1177316352.0 + "1": 971190784.0, + "2": 1232497664.0, + "3": 1232497664.0, + "4": 1232497664.0, + "5": 1232497664.0, + "6": 1232497664.0, + "7": 1232497664.0, + "8": 1232497664.0, + "9": 1232497664.0, + "10": 1232497664.0, + "11": 1232498688.0, + "12": 1232498688.0, + "13": 1232498688.0, + "14": 1232498688.0, + "15": 1232498688.0, + "16": 1232498688.0, + "17": 1232498688.0, + "18": 1232498688.0, + "19": 1232498688.0, + "20": 1232498688.0, + "21": 1232498688.0, + "22": 1232498688.0, + "23": 1232498688.0, + "24": 1232498688.0, + "25": 1232498688.0, + "26": 1232498688.0, + "27": 1232498688.0, + "28": 1232498688.0, + "29": 1232498688.0, + "30": 1232499200.0, + "31": 1232499200.0, + "32": 1232499200.0, + "33": 1232499200.0, + "34": 1232499200.0, + "35": 1232499200.0, + "36": 1232499200.0, + "37": 1232499200.0, + "38": 1232499200.0, + "39": 1232499200.0, + "40": 1232499200.0, + "41": 1232499200.0, + "42": 1232499200.0, + "43": 1232499200.0, + "44": 1232499200.0, + "45": 1232499200.0, + "46": 1232499200.0, + "47": 1232499200.0, + "48": 1232499200.0, + "49": 1232499200.0, + "50": 1232499200.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.38339, - "3": 1.82517, - "4": 1.52689, - "5": 1.15859, - "6": 1.66678, - "7": 1.55786, - "8": 2.39358, - "9": 2.27873, - "10": 1.89866, - "11": 1.87264, - "12": 1.93649, - "13": 1.96768, - "14": 2.18369, - "15": 1.77257, - "16": 1.82404, - "17": 2.50838, - "18": 1.72981, - "19": 1.87305, - "20": 1.88896, - "21": 2.21039, - "22": 1.46131, - "23": 2.11176, - "24": 1.88896, - "25": 2.26135, - "26": 2.46814, - "27": 1.99314, - "28": 2.03385, - "29": 1.68086, - "30": 1.65779, - "31": 1.65574, - "32": 2.62807, - "33": 1.57794, - "34": 1.6315, - "35": 2.39842, - "36": 2.07391, - "37": 1.76618, - "38": 1.66546, - "39": 2.21354, - "40": 1.82935, - "41": 1.82561, - "42": 2.80846, - "43": 1.69614, - "44": 1.95422, - "45": 2.31591, - "46": 1.69895, - "47": 1.76076, - "48": 2.09921, - "49": 1.63754, - "50": 1.84687 + "2": 6.29853, + "3": 1.69246, + "4": 1.66257, + "5": 1.2402, + "6": 1.69335, + "7": 1.30211, + "8": 1.694, + "9": 1.79027, + "10": 1.79904, + "11": 1.31694, + "12": 1.52554, + "13": 1.56922, + "14": 1.64791, + "15": 1.36317, + "16": 1.73334, + "17": 2.30959, + "18": 1.42977, + "19": 1.58059, + "20": 1.32421, + "21": 1.7518, + "22": 1.37507, + "23": 1.62455, + "24": 1.50922, + "25": 1.69559, + "26": 2.18561, + "27": 1.87146, + "28": 1.62392, + "29": 1.38008, + "30": 1.41907, + "31": 1.2865, + "32": 2.34622, + "33": 1.55599, + "34": 1.40023, + "35": 2.08714, + "36": 1.45378, + "37": 1.24335, + "38": 1.50503, + "39": 1.7646, + "40": 1.51234, + "41": 1.74947, + "42": 2.19243, + "43": 1.37066, + "44": 1.64215, + "45": 1.88732, + "46": 1.46246, + "47": 1.40469, + "48": 1.64727, + "49": 1.48274, + "50": 1.93068 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json index 9f2a8f5252a..1c37470d3f2 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json @@ -6,53 +6,53 @@ "values": { "1": 10.9228, "2": 10.92259, - "3": 10.91129, - "4": 10.91262, - "5": 10.91229, - "6": 10.91115, - "7": 10.90761, - "8": 10.90486, - "9": 10.90445, - "10": 10.90371, - "11": 10.89156, - "12": 10.89897, - "13": 10.89488, + "3": 10.91131, + "4": 10.91261, + "5": 10.91231, + "6": 10.91112, + "7": 10.90763, + "8": 10.90489, + "9": 10.90448, + "10": 10.90372, + "11": 10.89157, + "12": 10.89898, + "13": 10.89492, "14": 10.88107, - "15": 10.85401, + "15": 10.85396, "16": 10.84622, - "17": 10.84596, + "17": 10.84592, "18": 10.83436, - "19": 10.83439, - "20": 10.73158, - "21": 10.71288, - "22": 10.71183, - "23": 10.70052, - "24": 10.66577, - "25": 10.66681, - "26": 10.65427, - "27": 10.59994, - "28": 10.53048, - "29": 10.50772, - "30": 10.49516, - "31": 10.47433, - "32": 10.4633, - "33": 10.42698, + "19": 10.83435, + "20": 10.73163, + "21": 10.71287, + "22": 10.71185, + "23": 10.70053, + "24": 10.66578, + "25": 10.66683, + "26": 10.65428, + "27": 10.59989, + "28": 10.53051, + "29": 10.50775, + "30": 10.49518, + "31": 10.47431, + "32": 10.46336, + "33": 10.42695, "34": 10.39676, - "35": 10.39597, - "36": 10.38022, - "37": 10.33527, - "38": 10.34443, - "39": 10.30848, - "40": 10.30596, - "41": 10.27175, - "42": 10.25554, - "43": 10.22764, - "44": 10.20321, - "45": 10.22047, - "46": 10.18449, - "47": 10.16912, - "48": 10.11941, - "49": 10.1263, + "35": 10.39601, + "36": 10.38026, + "37": 10.33528, + "38": 10.34442, + "39": 10.30845, + "40": 10.30599, + "41": 10.27178, + "42": 10.25552, + "43": 10.22763, + "44": 10.20322, + "45": 10.2205, + "46": 10.18447, + "47": 10.16913, + "48": 10.11938, + "49": 10.12631, "50": 10.13123 } }, @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 618.0, - "2": 609.0, - "3": 605.0, - "4": 631.0, - "5": 619.0, - "6": 616.0, - "7": 696.0, - "8": 613.0, - "9": 624.0, - "10": 661.0, - "11": 605.0, - "12": 597.0, - "13": 621.0, - "14": 662.0, - "15": 571.0, - "16": 614.0, - "17": 653.0, - "18": 629.0, - "19": 620.0, - "20": 639.0, - "21": 679.0, - "22": 603.0, - "23": 618.0, - "24": 650.0, - "25": 624.0, - "26": 663.0, - "27": 679.0, - "28": 728.0, - "29": 671.0, - "30": 717.0, - "31": 731.0, - "32": 748.0, - "33": 757.0, - "34": 726.0, - "35": 759.0, - "36": 758.0, - "37": 770.0, - "38": 819.0, - "39": 886.0, - "40": 840.0, - "41": 936.0, - "42": 784.0, - "43": 887.0, - "44": 849.0, - "45": 939.0, - "46": 925.0, - "47": 929.0, - "48": 948.0, - "49": 1027.0, - "50": 1050.0 + "1": 640.0, + "2": 605.0, + "3": 603.0, + "4": 605.0, + "5": 606.0, + "6": 585.0, + "7": 697.0, + "8": 602.0, + "9": 630.0, + "10": 624.0, + "11": 678.0, + "12": 583.0, + "13": 620.0, + "14": 626.0, + "15": 605.0, + "16": 598.0, + "17": 634.0, + "18": 601.0, + "19": 626.0, + "20": 565.0, + "21": 683.0, + "22": 619.0, + "23": 648.0, + "24": 699.0, + "25": 628.0, + "26": 657.0, + "27": 653.0, + "28": 642.0, + "29": 760.0, + "30": 705.0, + "31": 753.0, + "32": 740.0, + "33": 718.0, + "34": 766.0, + "35": 809.0, + "36": 722.0, + "37": 765.0, + "38": 782.0, + "39": 838.0, + "40": 915.0, + "41": 918.0, + "42": 739.0, + "43": 847.0, + "44": 864.0, + "45": 968.0, + "46": 940.0, + "47": 893.0, + "48": 1004.0, + "49": 1015.0, + "50": 1028.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 463110656.0, - "2": 463110656.0, - "3": 463110656.0, - "4": 463110656.0, - "5": 463110656.0, - "6": 463110656.0, - "7": 463110656.0, - "8": 463110656.0, - "9": 463110656.0, - "10": 463110656.0, - "11": 463110656.0, - "12": 463110656.0, - "13": 463110656.0, - "14": 463110656.0, - "15": 463110656.0, - "16": 463110656.0, - "17": 463110656.0, - "18": 463110656.0, - "19": 463110656.0, - "20": 463110656.0, - "21": 463110656.0, - "22": 463110656.0, - "23": 463110656.0, - "24": 463110656.0, - "25": 463110656.0, - "26": 463110656.0, - "27": 463110656.0, - "28": 463110656.0, - "29": 463110656.0, - "30": 463110656.0, - "31": 463110656.0, - "32": 463110656.0, - "33": 463110656.0, - "34": 463110656.0, - "35": 463110656.0, - "36": 463110656.0, - "37": 463110656.0, - "38": 463110656.0, - "39": 463110656.0, - "40": 463110656.0, - "41": 463110656.0, - "42": 463110656.0, - "43": 463110656.0, - "44": 463110656.0, - "45": 463110656.0, - "46": 463110656.0, - "47": 463110656.0, - "48": 463110656.0, - "49": 463110656.0, - "50": 463110656.0 + "1": 512262656.0, + "2": 512262656.0, + "3": 512262656.0, + "4": 512262656.0, + "5": 512262656.0, + "6": 512262656.0, + "7": 512262656.0, + "8": 512262656.0, + "9": 512262656.0, + "10": 512262656.0, + "11": 512262656.0, + "12": 512262656.0, + "13": 512262656.0, + "14": 512262656.0, + "15": 512262656.0, + "16": 512262656.0, + "17": 512262656.0, + "18": 512262656.0, + "19": 512262656.0, + "20": 512262656.0, + "21": 512262656.0, + "22": 512262656.0, + "23": 512262656.0, + "24": 512262656.0, + "25": 512262656.0, + "26": 512262656.0, + "27": 512262656.0, + "28": 512262656.0, + "29": 512262656.0, + "30": 512262656.0, + "31": 512262656.0, + "32": 512262656.0, + "33": 512262656.0, + "34": 512262656.0, + "35": 512262656.0, + "36": 512262656.0, + "37": 512262656.0, + "38": 512262656.0, + "39": 512262656.0, + "40": 512262656.0, + "41": 512262656.0, + "42": 512262656.0, + "43": 512262656.0, + "44": 512262656.0, + "45": 512262656.0, + "46": 512262656.0, + "47": 512262656.0, + "48": 512262656.0, + "49": 512262656.0, + "50": 512262656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 705635840.0, - "2": 887673856.0, - "3": 887674368.0, - "4": 887674368.0, - "5": 887674368.0, - "6": 887674368.0, - "7": 887674368.0, - "8": 887674368.0, - "9": 887674368.0, - "10": 887674368.0, - "11": 887674368.0, - "12": 887674368.0, - "13": 887674368.0, - "14": 887674368.0, - "15": 887674368.0, - "16": 887674368.0, - "17": 887674368.0, - "18": 887674368.0, - "19": 887674368.0, - "20": 887674368.0, - "21": 887674368.0, - "22": 887674368.0, - "23": 887674368.0, - "24": 887674368.0, - "25": 887674368.0, - "26": 887674368.0, - "27": 887674368.0, - "28": 887674368.0, - "29": 887674368.0, - "30": 887674368.0, - "31": 887674368.0, - "32": 887674368.0, - "33": 887674368.0, - "34": 887674368.0, - "35": 887674368.0, - "36": 887674368.0, - "37": 887674368.0, - "38": 887674368.0, - "39": 887674368.0, - "40": 887674368.0, - "41": 887674368.0, - "42": 887674368.0, - "43": 887674368.0, - "44": 887674368.0, - "45": 887674368.0, - "46": 887674368.0, - "47": 887674368.0, - "48": 887674368.0, - "49": 887674368.0, - "50": 887674368.0 + "1": 754656768.0, + "2": 935777792.0, + "3": 935777792.0, + "4": 935777792.0, + "5": 935777792.0, + "6": 935777792.0, + "7": 935777792.0, + "8": 935777792.0, + "9": 935777792.0, + "10": 935777792.0, + "11": 935777792.0, + "12": 935777792.0, + "13": 935777792.0, + "14": 935777792.0, + "15": 935777792.0, + "16": 935777792.0, + "17": 935777792.0, + "18": 935777792.0, + "19": 935777792.0, + "20": 935777792.0, + "21": 935777792.0, + "22": 935777792.0, + "23": 935777792.0, + "24": 935777792.0, + "25": 935777792.0, + "26": 935777792.0, + "27": 935777792.0, + "28": 935777792.0, + "29": 935777792.0, + "30": 935777792.0, + "31": 935777792.0, + "32": 935777792.0, + "33": 935777792.0, + "34": 935777792.0, + "35": 935777792.0, + "36": 935777792.0, + "37": 935777792.0, + "38": 935777792.0, + "39": 935777792.0, + "40": 936824320.0, + "41": 936824320.0, + "42": 936824320.0, + "43": 936824320.0, + "44": 936824320.0, + "45": 936824320.0, + "46": 936824320.0, + "47": 936824320.0, + "48": 936824320.0, + "49": 936824320.0, + "50": 936824320.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.68242, - "3": 0.47666, - "4": 0.47014, - "5": 0.4701, - "6": 0.46618, - "7": 0.46921, - "8": 0.468, - "9": 0.46693, - "10": 0.46488, - "11": 0.46501, - "12": 0.46739, - "13": 0.47229, - "14": 0.46922, - "15": 0.47226, - "16": 0.46565, - "17": 0.46648, - "18": 0.46936, - "19": 0.46571, - "20": 0.47014, - "21": 0.46273, - "22": 0.46904, - "23": 0.46903, - "24": 0.47115, - "25": 0.46896, - "26": 0.46981, - "27": 0.46966, - "28": 0.47044, - "29": 0.46905, - "30": 0.47067, - "31": 0.47603, - "32": 0.46745, - "33": 0.4708, - "34": 0.56034, - "35": 0.47503, - "36": 0.46976, - "37": 0.46841, - "38": 0.46856, - "39": 0.47059, - "40": 0.4626, - "41": 0.46397, - "42": 0.46464, - "43": 0.46359, - "44": 0.46477, - "45": 0.4656, - "46": 0.46657, - "47": 0.46007, - "48": 0.46906, - "49": 0.47072, - "50": 0.47141 + "2": 11.76327, + "3": 0.46264, + "4": 0.4106, + "5": 0.40984, + "6": 0.41162, + "7": 0.4124, + "8": 0.40994, + "9": 0.41035, + "10": 0.40271, + "11": 0.41349, + "12": 0.41067, + "13": 0.40925, + "14": 0.40944, + "15": 0.41204, + "16": 0.40355, + "17": 0.4073, + "18": 0.4031, + "19": 0.40577, + "20": 0.4094, + "21": 0.40961, + "22": 0.41084, + "23": 0.41059, + "24": 0.4098, + "25": 0.40923, + "26": 0.40866, + "27": 0.40914, + "28": 0.40873, + "29": 0.40627, + "30": 0.40668, + "31": 0.40629, + "32": 0.41134, + "33": 0.41056, + "34": 0.41059, + "35": 0.40799, + "36": 0.41083, + "37": 0.41042, + "38": 0.41516, + "39": 0.41417, + "40": 0.4081, + "41": 0.41178, + "42": 0.41891, + "43": 0.41161, + "44": 0.42215, + "45": 0.41792, + "46": 0.41096, + "47": 0.41818, + "48": 0.41468, + "49": 0.41516, + "50": 0.41743 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json index 1304bf33e11..7b7b58d80a0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 640073216.0, - "2": 640073216.0, - "3": 640073216.0, - "4": 640073216.0, - "5": 640073216.0, - "6": 640073216.0, - "7": 640073216.0, - "8": 640073216.0, - "9": 640073216.0, - "10": 640073216.0, - "11": 640073216.0, - "12": 640073216.0, - "13": 640073216.0, - "14": 640073216.0, - "15": 640073216.0, - "16": 640073216.0, - "17": 640073216.0, - "18": 640073216.0, - "19": 640073216.0, - "20": 640073216.0, - "21": 640073216.0, - "22": 640073216.0, - "23": 640073216.0, - "24": 640073216.0, - "25": 640073216.0, - "26": 640073216.0, - "27": 640073216.0, - "28": 640073216.0, - "29": 640073216.0, - "30": 640073216.0, - "31": 640073216.0, - "32": 640073216.0, - "33": 640073216.0, - "34": 640073216.0, - "35": 640073216.0, - "36": 640073216.0, - "37": 640073216.0, - "38": 640073216.0, - "39": 640073216.0, - "40": 640073216.0, - "41": 640073216.0, - "42": 640073216.0, - "43": 640073216.0, - "44": 640073216.0, - "45": 640073216.0, - "46": 640073216.0, - "47": 640073216.0, - "48": 640073216.0, - "49": 640073216.0, - "50": 640073216.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 909846528.0, - "2": 1170895360.0, - "3": 1183476736.0, - "4": 1183478272.0, - "5": 1183478272.0, - "6": 1183739904.0, - "7": 1183739904.0, - "8": 1183739904.0, - "9": 1183739904.0, - "10": 1183739904.0, - "11": 1183739904.0, - "12": 1183739904.0, - "13": 1183739904.0, - "14": 1183739904.0, - "15": 1184393216.0, - "16": 1184393216.0, - "17": 1184393216.0, - "18": 1184393216.0, - "19": 1184393216.0, - "20": 1184393216.0, - "21": 1184393216.0, - "22": 1184393216.0, - "23": 1184393216.0, - "24": 1184393216.0, - "25": 1184393216.0, - "26": 1184393216.0, - "27": 1184393216.0, - "28": 1184393216.0, - "29": 1184393216.0, - "30": 1184393216.0, - "31": 1184393216.0, - "32": 1184393216.0, - "33": 1184393216.0, - "34": 1184393216.0, - "35": 1184393216.0, - "36": 1184393216.0, - "37": 1184393216.0, - "38": 1184393216.0, - "39": 1184393216.0, - "40": 1184393216.0, - "41": 1184393216.0, - "42": 1184393216.0, - "43": 1184393216.0, - "44": 1184393216.0, - "45": 1184393216.0, - "46": 1184393216.0, - "47": 1184393216.0, - "48": 1184393216.0, - "49": 1184393216.0, - "50": 1184393216.0 + "1": 968045056.0, + "2": 1224371200.0, + "3": 1224371200.0, + "4": 1227515392.0, + "5": 1227516928.0, + "6": 1227516928.0, + "7": 1229612544.0, + "8": 1229613568.0, + "9": 1229613568.0, + "10": 1229613568.0, + "11": 1229613568.0, + "12": 1229613568.0, + "13": 1229613568.0, + "14": 1229613568.0, + "15": 1229613568.0, + "16": 1229613568.0, + "17": 1229613568.0, + "18": 1229613568.0, + "19": 1229613568.0, + "20": 1229613568.0, + "21": 1229613568.0, + "22": 1229613568.0, + "23": 1229613568.0, + "24": 1229613568.0, + "25": 1229613568.0, + "26": 1231712768.0, + "27": 1231712768.0, + "28": 1231712768.0, + "29": 1231712768.0, + "30": 1231712768.0, + "31": 1231712768.0, + "32": 1231712768.0, + "33": 1231712768.0, + "34": 1231712768.0, + "35": 1231712768.0, + "36": 1231712768.0, + "37": 1231712768.0, + "38": 1231712768.0, + "39": 1231712768.0, + "40": 1231712768.0, + "41": 1231712768.0, + "42": 1231712768.0, + "43": 1231712768.0, + "44": 1231712768.0, + "45": 1231712768.0, + "46": 1231712768.0, + "47": 1231712768.0, + "48": 1231712768.0, + "49": 1231712768.0, + "50": 1231712768.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.12844, - "3": 2.21371, - "4": 1.92237, - "5": 1.6034, - "6": 2.00558, - "7": 1.38797, - "8": 2.38842, - "9": 2.43381, - "10": 1.84294, - "11": 2.16314, - "12": 2.01399, - "13": 1.99068, - "14": 2.28159, - "15": 1.72273, - "16": 2.05062, - "17": 2.2891, - "18": 1.71064, - "19": 1.89348, - "20": 1.98018, - "21": 1.84498, - "22": 1.66919, - "23": 2.07264, - "24": 1.88944, - "25": 2.23371, - "26": 2.25357, - "27": 1.75533, - "28": 1.69791, - "29": 1.48707, - "30": 1.64585, - "31": 1.62523, - "32": 2.5903, - "33": 1.64794, - "34": 1.52985, - "35": 2.35537, - "36": 2.25158, - "37": 1.77292, - "38": 1.55473, - "39": 2.04974, - "40": 1.99804, - "41": 1.99066, - "42": 2.49355, - "43": 1.53976, - "44": 2.21525, - "45": 2.01832, - "46": 1.83985, - "47": 1.62482, - "48": 1.97152, - "49": 1.6326, - "50": 1.95741 + "2": 6.73272, + "3": 1.60606, + "4": 1.77261, + "5": 1.47159, + "6": 1.8026, + "7": 1.37628, + "8": 1.77966, + "9": 2.20208, + "10": 1.65925, + "11": 2.07495, + "12": 1.80485, + "13": 1.75721, + "14": 2.23394, + "15": 1.75862, + "16": 1.4649, + "17": 1.62767, + "18": 1.66978, + "19": 1.9783, + "20": 1.72044, + "21": 2.13668, + "22": 1.31353, + "23": 1.81191, + "24": 1.34626, + "25": 1.94929, + "26": 1.84221, + "27": 1.52342, + "28": 1.76785, + "29": 1.31145, + "30": 1.48904, + "31": 1.23676, + "32": 2.37774, + "33": 1.49483, + "34": 1.27551, + "35": 1.75157, + "36": 1.47533, + "37": 1.26464, + "38": 1.04308, + "39": 1.60063, + "40": 1.73087, + "41": 1.43602, + "42": 1.95513, + "43": 1.46115, + "44": 1.39775, + "45": 1.568, + "46": 0.98454, + "47": 1.55736, + "48": 1.7641, + "49": 1.39153, + "50": 1.59473 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json index c4271eff02a..2836fdc69fc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json @@ -6,53 +6,53 @@ "values": { "1": 10.9228, "2": 10.92259, - "3": 10.91129, - "4": 10.91262, - "5": 10.91229, - "6": 10.91115, - "7": 10.90761, - "8": 10.90486, - "9": 10.90445, - "10": 10.90371, - "11": 10.89156, - "12": 10.89897, - "13": 10.89488, + "3": 10.91131, + "4": 10.91261, + "5": 10.91231, + "6": 10.91112, + "7": 10.90763, + "8": 10.90489, + "9": 10.90448, + "10": 10.90372, + "11": 10.89157, + "12": 10.89898, + "13": 10.89492, "14": 10.88107, - "15": 10.85401, + "15": 10.85396, "16": 10.84622, - "17": 10.84596, + "17": 10.84592, "18": 10.83436, - "19": 10.83439, - "20": 10.73158, - "21": 10.71288, - "22": 10.71183, - "23": 10.70052, - "24": 10.66577, - "25": 10.66681, - "26": 10.65427, - "27": 10.59994, - "28": 10.53048, - "29": 10.50772, - "30": 10.49516, - "31": 10.47433, - "32": 10.4633, - "33": 10.42698, + "19": 10.83435, + "20": 10.73163, + "21": 10.71287, + "22": 10.71185, + "23": 10.70053, + "24": 10.66578, + "25": 10.66683, + "26": 10.65428, + "27": 10.59989, + "28": 10.53051, + "29": 10.50775, + "30": 10.49518, + "31": 10.47431, + "32": 10.46336, + "33": 10.42695, "34": 10.39676, - "35": 10.39597, - "36": 10.38022, - "37": 10.33527, - "38": 10.34443, - "39": 10.30848, - "40": 10.30596, - "41": 10.27175, - "42": 10.25554, - "43": 10.22764, - "44": 10.20321, - "45": 10.22047, - "46": 10.18449, - "47": 10.16912, - "48": 10.11941, - "49": 10.1263, + "35": 10.39601, + "36": 10.38026, + "37": 10.33528, + "38": 10.34442, + "39": 10.30845, + "40": 10.30599, + "41": 10.27178, + "42": 10.25552, + "43": 10.22763, + "44": 10.20322, + "45": 10.2205, + "46": 10.18447, + "47": 10.16913, + "48": 10.11938, + "49": 10.12631, "50": 10.13123 } }, @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 618.0, - "2": 609.0, - "3": 605.0, - "4": 631.0, - "5": 619.0, - "6": 616.0, - "7": 696.0, - "8": 613.0, - "9": 624.0, - "10": 661.0, - "11": 605.0, - "12": 597.0, - "13": 621.0, - "14": 662.0, - "15": 571.0, - "16": 614.0, - "17": 653.0, - "18": 629.0, - "19": 620.0, - "20": 639.0, - "21": 679.0, - "22": 603.0, - "23": 618.0, - "24": 650.0, - "25": 624.0, - "26": 663.0, - "27": 679.0, - "28": 728.0, - "29": 671.0, - "30": 717.0, - "31": 731.0, - "32": 748.0, - "33": 757.0, - "34": 726.0, - "35": 759.0, - "36": 758.0, - "37": 770.0, - "38": 819.0, - "39": 886.0, - "40": 840.0, - "41": 936.0, - "42": 784.0, - "43": 887.0, - "44": 849.0, - "45": 939.0, - "46": 925.0, - "47": 929.0, - "48": 948.0, - "49": 1027.0, - "50": 1050.0 + "1": 640.0, + "2": 605.0, + "3": 603.0, + "4": 605.0, + "5": 606.0, + "6": 585.0, + "7": 697.0, + "8": 602.0, + "9": 630.0, + "10": 624.0, + "11": 678.0, + "12": 583.0, + "13": 620.0, + "14": 626.0, + "15": 605.0, + "16": 598.0, + "17": 634.0, + "18": 601.0, + "19": 626.0, + "20": 565.0, + "21": 683.0, + "22": 619.0, + "23": 648.0, + "24": 699.0, + "25": 628.0, + "26": 657.0, + "27": 653.0, + "28": 642.0, + "29": 760.0, + "30": 705.0, + "31": 753.0, + "32": 740.0, + "33": 718.0, + "34": 766.0, + "35": 809.0, + "36": 722.0, + "37": 765.0, + "38": 782.0, + "39": 838.0, + "40": 915.0, + "41": 918.0, + "42": 739.0, + "43": 847.0, + "44": 864.0, + "45": 968.0, + "46": 940.0, + "47": 893.0, + "48": 1004.0, + "49": 1015.0, + "50": 1028.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 460882432.0, - "2": 460882432.0, - "3": 460882432.0, - "4": 460882432.0, - "5": 460882432.0, - "6": 460882432.0, - "7": 460882432.0, - "8": 460882432.0, - "9": 460882432.0, - "10": 460882432.0, - "11": 460882432.0, - "12": 460882432.0, - "13": 460882432.0, - "14": 460882432.0, - "15": 460882432.0, - "16": 460882432.0, - "17": 460882432.0, - "18": 460882432.0, - "19": 460882432.0, - "20": 460882432.0, - "21": 460882432.0, - "22": 460882432.0, - "23": 460882432.0, - "24": 460882432.0, - "25": 460882432.0, - "26": 460882432.0, - "27": 460882432.0, - "28": 460882432.0, - "29": 460882432.0, - "30": 460882432.0, - "31": 460882432.0, - "32": 460882432.0, - "33": 460882432.0, - "34": 460882432.0, - "35": 460882432.0, - "36": 460882432.0, - "37": 460882432.0, - "38": 460882432.0, - "39": 460882432.0, - "40": 460882432.0, - "41": 460882432.0, - "42": 460882432.0, - "43": 460882432.0, - "44": 460882432.0, - "45": 460882432.0, - "46": 460882432.0, - "47": 460882432.0, - "48": 460882432.0, - "49": 460882432.0, - "50": 460882432.0 + "1": 512262656.0, + "2": 512262656.0, + "3": 512262656.0, + "4": 512262656.0, + "5": 512262656.0, + "6": 512262656.0, + "7": 512262656.0, + "8": 512262656.0, + "9": 512262656.0, + "10": 512262656.0, + "11": 512262656.0, + "12": 512262656.0, + "13": 512262656.0, + "14": 512262656.0, + "15": 512262656.0, + "16": 512262656.0, + "17": 512262656.0, + "18": 512262656.0, + "19": 512262656.0, + "20": 512262656.0, + "21": 512262656.0, + "22": 512262656.0, + "23": 512262656.0, + "24": 512262656.0, + "25": 512262656.0, + "26": 512262656.0, + "27": 512262656.0, + "28": 512262656.0, + "29": 512262656.0, + "30": 512262656.0, + "31": 512262656.0, + "32": 512262656.0, + "33": 512262656.0, + "34": 512262656.0, + "35": 512262656.0, + "36": 512262656.0, + "37": 512262656.0, + "38": 512262656.0, + "39": 512262656.0, + "40": 512262656.0, + "41": 512262656.0, + "42": 512262656.0, + "43": 512262656.0, + "44": 512262656.0, + "45": 512262656.0, + "46": 512262656.0, + "47": 512262656.0, + "48": 512262656.0, + "49": 512262656.0, + "50": 512262656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 705635840.0, - "2": 883348992.0, - "3": 883348992.0, - "4": 883348992.0, - "5": 883348992.0, - "6": 883348992.0, - "7": 883348992.0, - "8": 883348992.0, - "9": 883348992.0, - "10": 883348992.0, - "11": 883348992.0, - "12": 883348992.0, - "13": 883348992.0, - "14": 883348992.0, - "15": 883348992.0, - "16": 883348992.0, - "17": 883348992.0, - "18": 883348992.0, - "19": 883348992.0, - "20": 883348992.0, - "21": 883348992.0, - "22": 883348992.0, - "23": 883348992.0, - "24": 883348992.0, - "25": 883348992.0, - "26": 883348992.0, - "27": 883348992.0, - "28": 883348992.0, - "29": 883348992.0, - "30": 883348992.0, - "31": 883348992.0, - "32": 883348992.0, - "33": 883348992.0, - "34": 883348992.0, - "35": 883348992.0, - "36": 883348992.0, - "37": 883348992.0, - "38": 883348992.0, - "39": 883348992.0, - "40": 883348992.0, - "41": 883348992.0, - "42": 883348992.0, - "43": 883348992.0, - "44": 883348992.0, - "45": 883348992.0, - "46": 883348992.0, - "47": 883348992.0, - "48": 883348992.0, - "49": 883348992.0, - "50": 883348992.0 + "1": 755703296.0, + "2": 935777792.0, + "3": 935777792.0, + "4": 936823808.0, + "5": 936823808.0, + "6": 936823808.0, + "7": 936826368.0, + "8": 936826368.0, + "9": 936826368.0, + "10": 936826368.0, + "11": 936826368.0, + "12": 936826368.0, + "13": 936826368.0, + "14": 936826368.0, + "15": 936826368.0, + "16": 936826368.0, + "17": 936826368.0, + "18": 936826368.0, + "19": 936826368.0, + "20": 936826368.0, + "21": 936826368.0, + "22": 936826368.0, + "23": 936826368.0, + "24": 936826368.0, + "25": 936826368.0, + "26": 936826368.0, + "27": 936826368.0, + "28": 936826368.0, + "29": 936826368.0, + "30": 936826368.0, + "31": 936826368.0, + "32": 936826368.0, + "33": 936826368.0, + "34": 936826368.0, + "35": 936826368.0, + "36": 936826368.0, + "37": 936826368.0, + "38": 936826368.0, + "39": 936826368.0, + "40": 936826368.0, + "41": 936826368.0, + "42": 936826368.0, + "43": 936826368.0, + "44": 936826368.0, + "45": 936826368.0, + "46": 936826368.0, + "47": 936826368.0, + "48": 936826368.0, + "49": 936826368.0, + "50": 936826368.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.12612, - "3": 0.47116, - "4": 0.45696, - "5": 0.46474, - "6": 0.4621, - "7": 0.46583, - "8": 0.46617, - "9": 0.46244, - "10": 0.46363, - "11": 0.46069, - "12": 0.46255, - "13": 0.46116, - "14": 0.46381, - "15": 0.46348, - "16": 0.46373, - "17": 0.4664, - "18": 0.45673, - "19": 0.4597, - "20": 0.45617, - "21": 0.45925, - "22": 0.4598, - "23": 0.46015, - "24": 0.45902, - "25": 0.46218, - "26": 0.46209, - "27": 0.46331, - "28": 0.46507, - "29": 0.46429, - "30": 0.46748, - "31": 0.46377, - "32": 0.46407, - "33": 0.46397, - "34": 0.46084, - "35": 0.46795, - "36": 0.46013, - "37": 0.46697, - "38": 0.46279, - "39": 0.46584, - "40": 0.46628, - "41": 0.46263, - "42": 0.46446, - "43": 0.46209, - "44": 0.46781, - "45": 0.45994, - "46": 0.46292, - "47": 0.4643, - "48": 0.4635, - "49": 0.46266, - "50": 0.46214 + "2": 13.80888, + "3": 0.46666, + "4": 0.41588, + "5": 0.41034, + "6": 0.41198, + "7": 0.41303, + "8": 0.40896, + "9": 0.41027, + "10": 0.40968, + "11": 0.41049, + "12": 0.41471, + "13": 0.42274, + "14": 0.41563, + "15": 0.41476, + "16": 0.41572, + "17": 0.41645, + "18": 0.41811, + "19": 0.41197, + "20": 0.41456, + "21": 0.41478, + "22": 0.4129, + "23": 0.40772, + "24": 0.40764, + "25": 0.41186, + "26": 0.40989, + "27": 0.41038, + "28": 0.40807, + "29": 0.40937, + "30": 0.41769, + "31": 0.42433, + "32": 0.41648, + "33": 0.42969, + "34": 0.41203, + "35": 0.41085, + "36": 0.41051, + "37": 0.40981, + "38": 0.40967, + "39": 0.41186, + "40": 0.41554, + "41": 0.41977, + "42": 0.41349, + "43": 0.41465, + "44": 0.41684, + "45": 0.41811, + "46": 0.41811, + "47": 0.40895, + "48": 0.4097, + "49": 0.41168, + "50": 0.41099 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json index 9b53019bf46..16b1bdecc61 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 640073216.0, - "2": 640073216.0, - "3": 640073216.0, - "4": 640073216.0, - "5": 640073216.0, - "6": 640073216.0, - "7": 640073216.0, - "8": 640073216.0, - "9": 640073216.0, - "10": 640073216.0, - "11": 640073216.0, - "12": 640073216.0, - "13": 640073216.0, - "14": 640073216.0, - "15": 640073216.0, - "16": 640073216.0, - "17": 640073216.0, - "18": 640073216.0, - "19": 640073216.0, - "20": 640073216.0, - "21": 640073216.0, - "22": 640073216.0, - "23": 640073216.0, - "24": 640073216.0, - "25": 640073216.0, - "26": 640073216.0, - "27": 640073216.0, - "28": 640073216.0, - "29": 640073216.0, - "30": 640073216.0, - "31": 640073216.0, - "32": 640073216.0, - "33": 640073216.0, - "34": 640073216.0, - "35": 640073216.0, - "36": 640073216.0, - "37": 640073216.0, - "38": 640073216.0, - "39": 640073216.0, - "40": 640073216.0, - "41": 640073216.0, - "42": 640073216.0, - "43": 640073216.0, - "44": 640073216.0, - "45": 640073216.0, - "46": 640073216.0, - "47": 640073216.0, - "48": 640073216.0, - "49": 640073216.0, - "50": 640073216.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 910371328.0, - "2": 1183478272.0, - "3": 1184393216.0, - "4": 1184393216.0, - "5": 1184393216.0, - "6": 1184393216.0, - "7": 1184393216.0, - "8": 1184393216.0, - "9": 1184393216.0, - "10": 1184394752.0, - "11": 1184394752.0, - "12": 1184394752.0, - "13": 1184394752.0, - "14": 1184394752.0, - "15": 1184394752.0, - "16": 1184394752.0, - "17": 1184394752.0, - "18": 1184394752.0, - "19": 1184394752.0, - "20": 1184394752.0, - "21": 1184394752.0, - "22": 1184394752.0, - "23": 1184394752.0, - "24": 1184394752.0, - "25": 1184394752.0, - "26": 1184394752.0, - "27": 1184394752.0, - "28": 1184394752.0, - "29": 1184394752.0, - "30": 1184394752.0, - "31": 1184394752.0, - "32": 1184394752.0, - "33": 1184394752.0, - "34": 1184394752.0, - "35": 1184394752.0, - "36": 1184394752.0, - "37": 1184394752.0, - "38": 1184394752.0, - "39": 1184394752.0, - "40": 1184394752.0, - "41": 1184394752.0, - "42": 1184394752.0, - "43": 1184394752.0, - "44": 1184394752.0, - "45": 1184394752.0, - "46": 1184394752.0, - "47": 1184394752.0, - "48": 1184394752.0, - "49": 1184394752.0, - "50": 1184394752.0 + "1": 971190784.0, + "2": 1231712768.0, + "3": 1232497664.0, + "4": 1232497664.0, + "5": 1232497664.0, + "6": 1232497664.0, + "7": 1232497664.0, + "8": 1232497664.0, + "9": 1232498176.0, + "10": 1232498176.0, + "11": 1232498176.0, + "12": 1232498176.0, + "13": 1232498176.0, + "14": 1232498176.0, + "15": 1232498176.0, + "16": 1232498176.0, + "17": 1232498176.0, + "18": 1232498176.0, + "19": 1232498176.0, + "20": 1232498176.0, + "21": 1232498176.0, + "22": 1232498176.0, + "23": 1232498176.0, + "24": 1232498176.0, + "25": 1232498176.0, + "26": 1232498176.0, + "27": 1232498176.0, + "28": 1232499200.0, + "29": 1232499200.0, + "30": 1232499200.0, + "31": 1232499200.0, + "32": 1232499200.0, + "33": 1232499200.0, + "34": 1232499200.0, + "35": 1232499200.0, + "36": 1232499200.0, + "37": 1232499200.0, + "38": 1232499200.0, + "39": 1232499200.0, + "40": 1232499200.0, + "41": 1232499200.0, + "42": 1232499200.0, + "43": 1232499200.0, + "44": 1232499200.0, + "45": 1232499200.0, + "46": 1232499200.0, + "47": 1232499200.0, + "48": 1232499200.0, + "49": 1232499200.0, + "50": 1232499200.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.04512, - "3": 2.53622, - "4": 2.14187, - "5": 1.70192, - "6": 2.20905, - "7": 1.87249, - "8": 2.24231, - "9": 2.04395, - "10": 1.95361, - "11": 2.16036, - "12": 1.74772, - "13": 1.9621, - "14": 2.19091, - "15": 1.86195, - "16": 1.75546, - "17": 2.37569, - "18": 1.86685, - "19": 1.80379, - "20": 1.6236, - "21": 2.02964, - "22": 1.8257, - "23": 2.23803, - "24": 1.59366, - "25": 2.45307, - "26": 2.38901, - "27": 1.83894, - "28": 1.92709, - "29": 1.52188, - "30": 1.73958, - "31": 1.65113, - "32": 3.11449, - "33": 1.54333, - "34": 1.71238, - "35": 2.41273, - "36": 2.26339, - "37": 1.89629, - "38": 1.57503, - "39": 2.4608, - "40": 2.11154, - "41": 2.11892, - "42": 2.642, - "43": 1.72126, - "44": 1.96065, - "45": 2.43263, - "46": 1.83922, - "47": 2.06797, - "48": 1.96396, - "49": 1.58463, - "50": 2.07216 + "2": 6.86967, + "3": 2.24353, + "4": 1.63766, + "5": 1.24249, + "6": 1.87386, + "7": 1.43654, + "8": 2.25014, + "9": 1.92616, + "10": 1.82329, + "11": 1.68641, + "12": 1.51359, + "13": 2.02324, + "14": 2.10997, + "15": 1.45322, + "16": 1.80307, + "17": 1.84803, + "18": 1.48852, + "19": 1.85293, + "20": 1.89938, + "21": 2.30027, + "22": 1.75518, + "23": 2.37999, + "24": 1.65086, + "25": 2.20202, + "26": 2.44209, + "27": 1.91641, + "28": 1.77296, + "29": 1.48484, + "30": 1.78434, + "31": 1.50521, + "32": 2.7713, + "33": 1.555, + "34": 1.62012, + "35": 2.12464, + "36": 2.29617, + "37": 1.56504, + "38": 1.62969, + "39": 2.34023, + "40": 2.16557, + "41": 2.39115, + "42": 1.93937, + "43": 2.01504, + "44": 1.88409, + "45": 1.9813, + "46": 1.8044, + "47": 1.8945, + "48": 1.90407, + "49": 1.58766, + "50": 2.03815 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json index 4a57f36f101..3389a984287 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json @@ -6,53 +6,53 @@ "values": { "1": 10.9228, "2": 10.92259, - "3": 10.91129, - "4": 10.91262, - "5": 10.91229, - "6": 10.91115, - "7": 10.90761, - "8": 10.90486, - "9": 10.90445, - "10": 10.90371, - "11": 10.89156, - "12": 10.89897, - "13": 10.89488, + "3": 10.91131, + "4": 10.91261, + "5": 10.91231, + "6": 10.91112, + "7": 10.90763, + "8": 10.90489, + "9": 10.90448, + "10": 10.90372, + "11": 10.89157, + "12": 10.89898, + "13": 10.89492, "14": 10.88107, - "15": 10.85401, + "15": 10.85396, "16": 10.84622, - "17": 10.84596, + "17": 10.84592, "18": 10.83436, - "19": 10.83439, - "20": 10.73158, - "21": 10.71288, - "22": 10.71183, - "23": 10.70052, - "24": 10.66577, - "25": 10.66681, - "26": 10.65427, - "27": 10.59994, - "28": 10.53048, - "29": 10.50772, - "30": 10.49516, - "31": 10.47433, - "32": 10.4633, - "33": 10.42698, + "19": 10.83435, + "20": 10.73163, + "21": 10.71287, + "22": 10.71185, + "23": 10.70053, + "24": 10.66578, + "25": 10.66683, + "26": 10.65428, + "27": 10.59989, + "28": 10.53051, + "29": 10.50775, + "30": 10.49518, + "31": 10.47431, + "32": 10.46336, + "33": 10.42695, "34": 10.39676, - "35": 10.39597, - "36": 10.38022, - "37": 10.33527, - "38": 10.34443, - "39": 10.30848, - "40": 10.30596, - "41": 10.27175, - "42": 10.25554, - "43": 10.22764, - "44": 10.20321, - "45": 10.22047, - "46": 10.18449, - "47": 10.16912, - "48": 10.11941, - "49": 10.1263, + "35": 10.39601, + "36": 10.38026, + "37": 10.33528, + "38": 10.34442, + "39": 10.30845, + "40": 10.30599, + "41": 10.27178, + "42": 10.25552, + "43": 10.22763, + "44": 10.20322, + "45": 10.2205, + "46": 10.18447, + "47": 10.16913, + "48": 10.11938, + "49": 10.12631, "50": 10.13123 } }, @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 618.0, - "2": 609.0, - "3": 605.0, - "4": 631.0, - "5": 619.0, - "6": 616.0, - "7": 696.0, - "8": 613.0, - "9": 624.0, - "10": 661.0, - "11": 605.0, - "12": 597.0, - "13": 621.0, - "14": 662.0, - "15": 571.0, - "16": 614.0, - "17": 653.0, - "18": 629.0, - "19": 620.0, - "20": 639.0, - "21": 679.0, - "22": 603.0, - "23": 618.0, - "24": 650.0, - "25": 624.0, - "26": 663.0, - "27": 679.0, - "28": 728.0, - "29": 671.0, - "30": 717.0, - "31": 731.0, - "32": 748.0, - "33": 757.0, - "34": 726.0, - "35": 759.0, - "36": 758.0, - "37": 770.0, - "38": 819.0, - "39": 886.0, - "40": 840.0, - "41": 936.0, - "42": 784.0, - "43": 887.0, - "44": 849.0, - "45": 939.0, - "46": 925.0, - "47": 929.0, - "48": 948.0, - "49": 1027.0, - "50": 1050.0 + "1": 640.0, + "2": 605.0, + "3": 603.0, + "4": 605.0, + "5": 606.0, + "6": 585.0, + "7": 697.0, + "8": 602.0, + "9": 630.0, + "10": 624.0, + "11": 678.0, + "12": 583.0, + "13": 620.0, + "14": 626.0, + "15": 605.0, + "16": 598.0, + "17": 634.0, + "18": 601.0, + "19": 626.0, + "20": 565.0, + "21": 683.0, + "22": 619.0, + "23": 648.0, + "24": 699.0, + "25": 628.0, + "26": 657.0, + "27": 653.0, + "28": 642.0, + "29": 760.0, + "30": 705.0, + "31": 753.0, + "32": 740.0, + "33": 718.0, + "34": 766.0, + "35": 809.0, + "36": 722.0, + "37": 765.0, + "38": 782.0, + "39": 838.0, + "40": 915.0, + "41": 918.0, + "42": 739.0, + "43": 847.0, + "44": 864.0, + "45": 968.0, + "46": 940.0, + "47": 893.0, + "48": 1004.0, + "49": 1015.0, + "50": 1028.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 460882432.0, - "2": 460882432.0, - "3": 460882432.0, - "4": 460882432.0, - "5": 460882432.0, - "6": 460882432.0, - "7": 460882432.0, - "8": 460882432.0, - "9": 460882432.0, - "10": 460882432.0, - "11": 460882432.0, - "12": 460882432.0, - "13": 460882432.0, - "14": 460882432.0, - "15": 460882432.0, - "16": 460882432.0, - "17": 460882432.0, - "18": 460882432.0, - "19": 460882432.0, - "20": 460882432.0, - "21": 460882432.0, - "22": 460882432.0, - "23": 460882432.0, - "24": 460882432.0, - "25": 460882432.0, - "26": 460882432.0, - "27": 460882432.0, - "28": 460882432.0, - "29": 460882432.0, - "30": 460882432.0, - "31": 460882432.0, - "32": 460882432.0, - "33": 460882432.0, - "34": 460882432.0, - "35": 460882432.0, - "36": 460882432.0, - "37": 460882432.0, - "38": 460882432.0, - "39": 460882432.0, - "40": 460882432.0, - "41": 460882432.0, - "42": 460882432.0, - "43": 460882432.0, - "44": 460882432.0, - "45": 460882432.0, - "46": 460882432.0, - "47": 460882432.0, - "48": 460882432.0, - "49": 460882432.0, - "50": 460882432.0 + "1": 512262656.0, + "2": 512262656.0, + "3": 512262656.0, + "4": 512262656.0, + "5": 512262656.0, + "6": 512262656.0, + "7": 512262656.0, + "8": 512262656.0, + "9": 512262656.0, + "10": 512262656.0, + "11": 512262656.0, + "12": 512262656.0, + "13": 512262656.0, + "14": 512262656.0, + "15": 512262656.0, + "16": 512262656.0, + "17": 512262656.0, + "18": 512262656.0, + "19": 512262656.0, + "20": 512262656.0, + "21": 512262656.0, + "22": 512262656.0, + "23": 512262656.0, + "24": 512262656.0, + "25": 512262656.0, + "26": 512262656.0, + "27": 512262656.0, + "28": 512262656.0, + "29": 512262656.0, + "30": 512262656.0, + "31": 512262656.0, + "32": 512262656.0, + "33": 512262656.0, + "34": 512262656.0, + "35": 512262656.0, + "36": 512262656.0, + "37": 512262656.0, + "38": 512262656.0, + "39": 512262656.0, + "40": 512262656.0, + "41": 512262656.0, + "42": 512262656.0, + "43": 512262656.0, + "44": 512262656.0, + "45": 512262656.0, + "46": 512262656.0, + "47": 512262656.0, + "48": 512262656.0, + "49": 512262656.0, + "50": 512262656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 705635840.0, - "2": 883348992.0, - "3": 883348992.0, - "4": 883348992.0, - "5": 883348992.0, - "6": 883348992.0, - "7": 883348992.0, - "8": 883348992.0, - "9": 883348992.0, - "10": 883348992.0, - "11": 883348992.0, - "12": 883348992.0, - "13": 883348992.0, - "14": 883348992.0, - "15": 885446144.0, - "16": 885446144.0, - "17": 885446144.0, - "18": 885446144.0, - "19": 885446144.0, - "20": 885446144.0, - "21": 885446144.0, - "22": 885446144.0, - "23": 885446144.0, - "24": 885446144.0, - "25": 885446144.0, - "26": 885446144.0, - "27": 885446144.0, - "28": 885446144.0, - "29": 885446144.0, - "30": 885446144.0, - "31": 885446144.0, - "32": 885446144.0, - "33": 885446144.0, - "34": 885446144.0, - "35": 885446144.0, - "36": 885446144.0, - "37": 885446144.0, - "38": 885446144.0, - "39": 885446144.0, - "40": 885446144.0, - "41": 885446144.0, - "42": 885446144.0, - "43": 885446144.0, - "44": 885446144.0, - "45": 885446144.0, - "46": 885446144.0, - "47": 885446144.0, - "48": 885446144.0, - "49": 885446144.0, - "50": 885446144.0 + "1": 755702272.0, + "2": 936824832.0, + "3": 936824832.0, + "4": 936825344.0, + "5": 936825344.0, + "6": 936826368.0, + "7": 936826368.0, + "8": 936826368.0, + "9": 936826368.0, + "10": 936826368.0, + "11": 936826368.0, + "12": 936826368.0, + "13": 936826368.0, + "14": 936826368.0, + "15": 936826368.0, + "16": 936826368.0, + "17": 936826368.0, + "18": 936826368.0, + "19": 936826368.0, + "20": 936826368.0, + "21": 936826368.0, + "22": 936826368.0, + "23": 936826368.0, + "24": 936826368.0, + "25": 936826368.0, + "26": 936826368.0, + "27": 936826368.0, + "28": 936826368.0, + "29": 936826368.0, + "30": 936826368.0, + "31": 936826368.0, + "32": 936826368.0, + "33": 936826368.0, + "34": 936826368.0, + "35": 936826368.0, + "36": 936826368.0, + "37": 936826368.0, + "38": 936826368.0, + "39": 936826368.0, + "40": 936826368.0, + "41": 936826368.0, + "42": 936826368.0, + "43": 936826368.0, + "44": 936826368.0, + "45": 936826368.0, + "46": 936826368.0, + "47": 936826368.0, + "48": 936826368.0, + "49": 936826368.0, + "50": 936826368.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.04235, - "3": 0.46727, - "4": 0.46075, - "5": 0.45494, - "6": 0.46101, - "7": 0.4563, - "8": 0.4608, - "9": 0.46054, - "10": 0.46344, - "11": 0.46736, - "12": 0.46071, - "13": 0.46359, - "14": 0.46605, - "15": 0.48948, - "16": 0.545, - "17": 0.47712, - "18": 0.46481, - "19": 0.45835, - "20": 0.46391, - "21": 0.46108, - "22": 0.46263, - "23": 0.46337, - "24": 0.46079, - "25": 0.46635, - "26": 0.46086, - "27": 0.46454, - "28": 0.46677, - "29": 0.46398, - "30": 0.46369, - "31": 0.46754, - "32": 0.47096, - "33": 0.46833, - "34": 0.46976, - "35": 0.46318, - "36": 0.46606, - "37": 0.46492, - "38": 0.46156, - "39": 0.46517, - "40": 0.46836, - "41": 0.46157, - "42": 0.46254, - "43": 0.45857, - "44": 0.46295, - "45": 0.46147, - "46": 0.46616, - "47": 0.46344, - "48": 0.46686, - "49": 0.4648, - "50": 0.4618 + "2": 13.03533, + "3": 0.4681, + "4": 0.41149, + "5": 0.4094, + "6": 0.41291, + "7": 0.41049, + "8": 0.40486, + "9": 0.40943, + "10": 0.41226, + "11": 0.41386, + "12": 0.41348, + "13": 0.40816, + "14": 0.41347, + "15": 0.41767, + "16": 0.4632, + "17": 0.41395, + "18": 0.40673, + "19": 0.41084, + "20": 0.41363, + "21": 0.41282, + "22": 0.40962, + "23": 0.41087, + "24": 0.41447, + "25": 0.41794, + "26": 0.412, + "27": 0.4174, + "28": 0.41293, + "29": 0.40876, + "30": 0.40679, + "31": 0.40781, + "32": 0.40957, + "33": 0.4079, + "34": 0.40437, + "35": 0.40845, + "36": 0.40903, + "37": 0.40878, + "38": 0.40754, + "39": 0.41072, + "40": 0.41076, + "41": 0.4141, + "42": 0.47135, + "43": 0.4082, + "44": 0.4046, + "45": 0.40461, + "46": 0.41087, + "47": 0.41228, + "48": 0.42268, + "49": 0.41238, + "50": 0.41519 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json index b3da7a4d92b..a606b6cbbb0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 638631424.0, - "2": 638631424.0, - "3": 638631424.0, - "4": 638631424.0, - "5": 638631424.0, - "6": 638631424.0, - "7": 638631424.0, - "8": 638631424.0, - "9": 638631424.0, - "10": 638631424.0, - "11": 638631424.0, - "12": 638631424.0, - "13": 638631424.0, - "14": 638631424.0, - "15": 638631424.0, - "16": 638631424.0, - "17": 638631424.0, - "18": 638631424.0, - "19": 638631424.0, - "20": 638631424.0, - "21": 638631424.0, - "22": 638631424.0, - "23": 638631424.0, - "24": 638631424.0, - "25": 638631424.0, - "26": 638631424.0, - "27": 638631424.0, - "28": 638631424.0, - "29": 638631424.0, - "30": 638631424.0, - "31": 638631424.0, - "32": 638631424.0, - "33": 638631424.0, - "34": 638631424.0, - "35": 638631424.0, - "36": 638631424.0, - "37": 638631424.0, - "38": 638631424.0, - "39": 638631424.0, - "40": 638631424.0, - "41": 638631424.0, - "42": 638631424.0, - "43": 638631424.0, - "44": 638631424.0, - "45": 638631424.0, - "46": 638631424.0, - "47": 638631424.0, - "48": 638631424.0, - "49": 638631424.0, - "50": 638631424.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 911682560.0, - "2": 1170501120.0, - "3": 1177317376.0, - "4": 1177317376.0, - "5": 1177317376.0, - "6": 1177317376.0, - "7": 1177317376.0, - "8": 1177839104.0, - "9": 1177839104.0, - "10": 1177839104.0, - "11": 1177839104.0, - "12": 1177839104.0, - "13": 1177839104.0, - "14": 1177839104.0, - "15": 1177839104.0, - "16": 1177839104.0, - "17": 1177839104.0, - "18": 1177839104.0, - "19": 1177839104.0, - "20": 1177839104.0, - "21": 1177839104.0, - "22": 1177839104.0, - "23": 1177839104.0, - "24": 1177839104.0, - "25": 1177839104.0, - "26": 1177839104.0, - "27": 1177839104.0, - "28": 1177839104.0, - "29": 1177839104.0, - "30": 1177839104.0, - "31": 1177839104.0, - "32": 1177839104.0, - "33": 1177839104.0, - "34": 1177839104.0, - "35": 1177839104.0, - "36": 1177839104.0, - "37": 1177839104.0, - "38": 1180463616.0, - "39": 1180463616.0, - "40": 1180463616.0, - "41": 1180463616.0, - "42": 1180463616.0, - "43": 1180463616.0, - "44": 1180463616.0, - "45": 1180463616.0, - "46": 1180463616.0, - "47": 1180463616.0, - "48": 1180463616.0, - "49": 1180463616.0, - "50": 1180463616.0 + "1": 971190272.0, + "2": 1224370688.0, + "3": 1232497152.0, + "4": 1232497152.0, + "5": 1232499200.0, + "6": 1232499200.0, + "7": 1232499200.0, + "8": 1232499200.0, + "9": 1232499200.0, + "10": 1232499200.0, + "11": 1232499200.0, + "12": 1232499200.0, + "13": 1232499200.0, + "14": 1232499200.0, + "15": 1232499200.0, + "16": 1232499200.0, + "17": 1232499200.0, + "18": 1232499200.0, + "19": 1232499200.0, + "20": 1232499200.0, + "21": 1232499200.0, + "22": 1232499200.0, + "23": 1232499200.0, + "24": 1232499200.0, + "25": 1232499200.0, + "26": 1232499200.0, + "27": 1232499200.0, + "28": 1232499200.0, + "29": 1232499200.0, + "30": 1232499200.0, + "31": 1232499200.0, + "32": 1232499200.0, + "33": 1232499200.0, + "34": 1232499200.0, + "35": 1232499200.0, + "36": 1232499200.0, + "37": 1232499200.0, + "38": 1232499200.0, + "39": 1232499200.0, + "40": 1232499200.0, + "41": 1232499200.0, + "42": 1232499200.0, + "43": 1232499200.0, + "44": 1232499200.0, + "45": 1232499200.0, + "46": 1232499200.0, + "47": 1232499200.0, + "48": 1232499200.0, + "49": 1232499200.0, + "50": 1232499200.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.20715, - "3": 1.92588, - "4": 1.7209, - "5": 1.26617, - "6": 1.91225, - "7": 1.40677, - "8": 1.68753, - "9": 2.14463, - "10": 1.9968, - "11": 1.88521, - "12": 1.94345, - "13": 1.62284, - "14": 1.87583, - "15": 1.78938, - "16": 1.49862, - "17": 1.79407, - "18": 1.6566, - "19": 1.73373, - "20": 1.65499, - "21": 1.65561, - "22": 1.48916, - "23": 1.58948, - "24": 1.43527, - "25": 1.64856, - "26": 1.68885, - "27": 1.42867, - "28": 1.39778, - "29": 1.51816, - "30": 1.39869, - "31": 1.19871, - "32": 2.35599, - "33": 1.4144, - "34": 1.20483, - "35": 1.8763, - "36": 1.71344, - "37": 1.37824, - "38": 1.53973, - "39": 2.04209, - "40": 1.79838, - "41": 2.16606, - "42": 2.00263, - "43": 1.67464, - "44": 1.61536, - "45": 2.20218, - "46": 1.37692, - "47": 1.49279, - "48": 1.69211, - "49": 1.39893, - "50": 1.8565 + "2": 6.89935, + "3": 1.82402, + "4": 1.62848, + "5": 1.11134, + "6": 1.66222, + "7": 1.35401, + "8": 1.88271, + "9": 1.76297, + "10": 1.56757, + "11": 1.95272, + "12": 1.44267, + "13": 1.72311, + "14": 1.93663, + "15": 1.42099, + "16": 1.55366, + "17": 2.09512, + "18": 1.61161, + "19": 1.47203, + "20": 1.65304, + "21": 1.91931, + "22": 1.35659, + "23": 2.11315, + "24": 1.59829, + "25": 1.97857, + "26": 1.69211, + "27": 1.43236, + "28": 1.49528, + "29": 1.38867, + "30": 1.51773, + "31": 1.53258, + "32": 2.84364, + "33": 1.46328, + "34": 1.1651, + "35": 1.90399, + "36": 1.47119, + "37": 1.60361, + "38": 1.44311, + "39": 2.02277, + "40": 1.62682, + "41": 1.67621, + "42": 1.97363, + "43": 1.57264, + "44": 1.13627, + "45": 1.96962, + "46": 1.52676, + "47": 1.70833, + "48": 1.63824, + "49": 1.26972, + "50": 1.78978 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mhc/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mhc/golden_values_dev_dgx_h100.json index 8a856291495..51c04e49ae3 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mhc/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mhc/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.86153, - "2": 10.85471, - "3": 10.86694, + "1": 10.86155, + "2": 10.85472, + "3": 10.86697, "4": 10.84628, - "5": 10.88468, - "6": 10.8968, - "7": 10.87276, - "8": 10.86586, - "9": 10.86987, - "10": 10.83767, - "11": 10.89459, - "12": 10.87954, - "13": 10.87684, + "5": 10.8847, + "6": 10.89688, + "7": 10.87274, + "8": 10.86585, + "9": 10.86981, + "10": 10.83765, + "11": 10.89456, + "12": 10.87958, + "13": 10.87687, "14": 10.9036, - "15": 10.83112, - "16": 10.83449, - "17": 10.80068, - "18": 10.8207, - "19": 10.81463, - "20": 10.7181, - "21": 10.68634, - "22": 10.53197, - "23": 10.70487, - "24": 10.58555, - "25": 10.51897, - "26": 10.5849, - "27": 10.60104, - "28": 10.53536, - "29": 10.57116, - "30": 10.33242, - "31": 10.05836, - "32": 10.42794, - "33": 10.42026, - "34": 10.16983, - "35": 10.23071, - "36": 10.18758, - "37": 10.31243, - "38": 10.14212, - "39": 10.38137, - "40": 10.04847, - "41": 10.10333, - "42": 10.17154, - "43": 9.78293, - "44": 9.90957, - "45": 9.78507, - "46": 9.7689, + "15": 10.83116, + "16": 10.83448, + "17": 10.80064, + "18": 10.82072, + "19": 10.81455, + "20": 10.71812, + "21": 10.68626, + "22": 10.53195, + "23": 10.70489, + "24": 10.5856, + "25": 10.51905, + "26": 10.58493, + "27": 10.60103, + "28": 10.53538, + "29": 10.57117, + "30": 10.33243, + "31": 10.05838, + "32": 10.4279, + "33": 10.42024, + "34": 10.16988, + "35": 10.23073, + "36": 10.18762, + "37": 10.31244, + "38": 10.14213, + "39": 10.38141, + "40": 10.04848, + "41": 10.10338, + "42": 10.17158, + "43": 9.78294, + "44": 9.90959, + "45": 9.78513, + "46": 9.76889, "47": 10.10085, - "48": 9.80966, - "49": 9.48775, - "50": 9.86712 + "48": 9.80968, + "49": 9.4878, + "50": 9.86709 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1703.0, - "2": 1742.0, - "3": 1622.0, - "4": 1788.0, - "5": 1858.0, - "6": 1801.0, - "7": 1793.0, - "8": 1674.0, - "9": 1888.0, - "10": 1389.0, - "11": 1759.0, - "12": 1668.0, - "13": 1869.0, - "14": 1801.0, - "15": 1852.0, - "16": 1768.0, - "17": 1945.0, - "18": 1725.0, - "19": 1754.0, - "20": 1688.0, - "21": 1866.0, - "22": 1620.0, - "23": 2088.0, - "24": 1701.0, - "25": 1641.0, - "26": 1760.0, - "27": 1842.0, - "28": 2007.0, - "29": 1987.0, - "30": 1956.0, - "31": 1590.0, - "32": 1873.0, - "33": 2187.0, - "34": 1985.0, - "35": 1969.0, - "36": 1921.0, - "37": 2438.0, - "38": 2161.0, - "39": 2402.0, - "40": 2183.0, - "41": 2268.0, - "42": 2382.0, - "43": 2039.0, - "44": 2157.0, - "45": 2204.0, - "46": 2370.0, - "47": 2460.0, - "48": 2439.0, - "49": 2414.0, - "50": 2402.0 + "1": 1654.0, + "2": 1741.0, + "3": 1678.0, + "4": 1808.0, + "5": 1876.0, + "6": 1898.0, + "7": 1791.0, + "8": 1592.0, + "9": 1864.0, + "10": 1426.0, + "11": 1913.0, + "12": 1700.0, + "13": 1841.0, + "14": 1755.0, + "15": 1891.0, + "16": 1912.0, + "17": 1925.0, + "18": 1645.0, + "19": 1827.0, + "20": 1693.0, + "21": 1814.0, + "22": 1634.0, + "23": 1991.0, + "24": 1687.0, + "25": 1646.0, + "26": 1667.0, + "27": 1786.0, + "28": 2010.0, + "29": 2022.0, + "30": 1897.0, + "31": 1604.0, + "32": 1839.0, + "33": 2184.0, + "34": 2016.0, + "35": 1933.0, + "36": 1882.0, + "37": 2467.0, + "38": 2172.0, + "39": 2422.0, + "40": 2209.0, + "41": 2354.0, + "42": 2327.0, + "43": 1996.0, + "44": 2245.0, + "45": 2156.0, + "46": 2316.0, + "47": 2465.0, + "48": 2450.0, + "49": 2378.0, + "50": 2320.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 554817024.0, - "2": 554817024.0, + "1": 555865600.0, + "2": 555865600.0, "3": 555865600.0, "4": 555865600.0, - "5": 554817024.0, - "6": 554817024.0, - "7": 554817024.0, - "8": 554817024.0, - "9": 554817024.0, + "5": 555865600.0, + "6": 555865600.0, + "7": 555865600.0, + "8": 555865600.0, + "9": 555865600.0, "10": 555865600.0, - "11": 554817024.0, - "12": 554817024.0, - "13": 554817024.0, - "14": 554817024.0, - "15": 554817024.0, - "16": 554817024.0, - "17": 554817024.0, - "18": 554817024.0, - "19": 554817024.0, - "20": 554817024.0, - "21": 554817024.0, - "22": 554817024.0, - "23": 554817024.0, - "24": 554817024.0, - "25": 554817024.0, - "26": 554817024.0, - "27": 554817024.0, - "28": 554817024.0, - "29": 554817024.0, - "30": 554817024.0, - "31": 554817024.0, - "32": 554817024.0, - "33": 554817024.0, - "34": 554817024.0, - "35": 554817024.0, - "36": 554817024.0, + "11": 555865600.0, + "12": 555865600.0, + "13": 555865600.0, + "14": 555865600.0, + "15": 555865600.0, + "16": 555865600.0, + "17": 555865600.0, + "18": 555865600.0, + "19": 555865600.0, + "20": 555865600.0, + "21": 555865600.0, + "22": 555865600.0, + "23": 555865600.0, + "24": 555865600.0, + "25": 555865600.0, + "26": 555865600.0, + "27": 555865600.0, + "28": 555865600.0, + "29": 555865600.0, + "30": 555865600.0, + "31": 555865600.0, + "32": 555865600.0, + "33": 555865600.0, + "34": 555865600.0, + "35": 555865600.0, + "36": 555865600.0, "37": 555865600.0, - "38": 554817024.0, - "39": 554817024.0, - "40": 554817024.0, - "41": 554817024.0, - "42": 554817024.0, - "43": 554817024.0, - "44": 554817024.0, - "45": 554817024.0, + "38": 555865600.0, + "39": 555865600.0, + "40": 555865600.0, + "41": 555865600.0, + "42": 555865600.0, + "43": 555865600.0, + "44": 555865600.0, + "45": 555865600.0, "46": 555865600.0, - "47": 554817024.0, - "48": 554817024.0, - "49": 554817024.0, - "50": 554817024.0 + "47": 555865600.0, + "48": 555865600.0, + "49": 555865600.0, + "50": 555865600.0 } }, "mem-max-allocated-bytes": { @@ -175,7 +175,7 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1746300416.0, + "1": 1746955776.0, "2": 1934879232.0, "3": 1934879232.0, "4": 1934879232.0, @@ -200,14 +200,14 @@ "23": 1934879232.0, "24": 1934879232.0, "25": 1934879232.0, - "26": 1935927808.0, - "27": 1935927808.0, - "28": 1935927808.0, - "29": 1935927808.0, - "30": 1935927808.0, - "31": 1935927808.0, - "32": 1935927808.0, - "33": 1935927808.0, + "26": 1935927296.0, + "27": 1935927296.0, + "28": 1935927296.0, + "29": 1935927296.0, + "30": 1935927296.0, + "31": 1935927296.0, + "32": 1935927296.0, + "33": 1935927296.0, "34": 1935927808.0, "35": 1935927808.0, "36": 1935927808.0, @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 31.19119, - "3": 4.64254, - "4": 2.06235, - "5": 3.10645, - "6": 2.09979, - "7": 3.00592, - "8": 2.84917, - "9": 2.47663, - "10": 3.44241, - "11": 3.16127, - "12": 5.01739, - "13": 2.65652, - "14": 2.48699, - "15": 2.65524, - "16": 2.11061, - "17": 2.67153, - "18": 3.8657, - "19": 3.07496, - "20": 3.26175, - "21": 3.6739, - "22": 1.84516, - "23": 3.5372, - "24": 2.81029, - "25": 2.85832, - "26": 0.7157, - "27": 5.60633, - "28": 3.04555, - "29": 2.37229, - "30": 2.82131, - "31": 3.33094, - "32": 5.55589, - "33": 2.19105, - "34": 2.07484, - "35": 2.60419, - "36": 2.92689, - "37": 2.97485, - "38": 1.62047, - "39": 3.13391, - "40": 3.59651, - "41": 4.22308, - "42": 1.86597, - "43": 3.16598, - "44": 2.03267, - "45": 2.76972, - "46": 2.09152, - "47": 3.69723, - "48": 2.47382, - "49": 2.18467, - "50": 2.99757 + "2": 30.16457, + "3": 0.69331, + "4": 0.69236, + "5": 0.69011, + "6": 0.69431, + "7": 0.68739, + "8": 0.68382, + "9": 0.68467, + "10": 0.68772, + "11": 0.68456, + "12": 0.68193, + "13": 0.69128, + "14": 0.68205, + "15": 0.68208, + "16": 0.68686, + "17": 0.68599, + "18": 0.68567, + "19": 0.69039, + "20": 0.68754, + "21": 0.69693, + "22": 0.69584, + "23": 0.69108, + "24": 0.68558, + "25": 0.68807, + "26": 0.71618, + "27": 0.69064, + "28": 0.7033, + "29": 0.70268, + "30": 0.69756, + "31": 0.70299, + "32": 0.69313, + "33": 0.69455, + "34": 0.70319, + "35": 0.68861, + "36": 0.68994, + "37": 0.6851, + "38": 0.68942, + "39": 0.68364, + "40": 0.67981, + "41": 0.67851, + "42": 0.68629, + "43": 0.69028, + "44": 0.68641, + "45": 0.68513, + "46": 0.68077, + "47": 0.68391, + "48": 0.68376, + "49": 0.68312, + "50": 0.68105 } } -} +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json index 8d169938e56..1dfee8ef5c7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.95956, - "2": 10.95129, - "3": 10.95113, + "1": 10.95952, + "2": 10.95136, + "3": 10.95111, "4": 10.95155, - "5": 10.94875, - "6": 10.95221, + "5": 10.94874, + "6": 10.95222, "7": 10.95069, - "8": 10.95243, - "9": 10.94679, - "10": 10.94584, - "11": 10.94797, - "12": 10.95832, - "13": 10.94909, - "14": 10.95315, - "15": 10.95082, - "16": 10.94521, - "17": 10.95146, - "18": 10.95038, - "19": 10.95044, - "20": 10.9398, - "21": 10.94002, - "22": 10.93863, - "23": 10.92605, - "24": 10.92812, - "25": 10.92794, - "26": 10.92921, - "27": 10.92541, - "28": 10.89609, - "29": 10.88943, - "30": 10.88643, - "31": 10.88485, - "32": 10.87976, - "33": 10.87162, - "34": 10.87022, - "35": 10.86702, - "36": 10.86059, - "37": 10.86549, - "38": 10.83729, - "39": 10.80125, - "40": 10.80509, - "41": 10.79382, - "42": 10.78037, + "8": 10.9524, + "9": 10.94682, + "10": 10.94583, + "11": 10.94799, + "12": 10.95835, + "13": 10.94914, + "14": 10.9532, + "15": 10.95074, + "16": 10.94525, + "17": 10.95153, + "18": 10.95037, + "19": 10.95048, + "20": 10.93977, + "21": 10.94006, + "22": 10.93861, + "23": 10.9261, + "24": 10.92811, + "25": 10.92789, + "26": 10.92913, + "27": 10.92535, + "28": 10.89611, + "29": 10.88941, + "30": 10.88644, + "31": 10.88483, + "32": 10.87977, + "33": 10.87161, + "34": 10.87025, + "35": 10.86697, + "36": 10.86056, + "37": 10.86543, + "38": 10.83735, + "39": 10.80126, + "40": 10.80515, + "41": 10.79389, + "42": 10.78032, "43": 10.77822, - "44": 10.77233, - "45": 10.77153, - "46": 10.74969, - "47": 10.73893, - "48": 10.72036, - "49": 10.73373, - "50": 10.71459 + "44": 10.77239, + "45": 10.7715, + "46": 10.7498, + "47": 10.73894, + "48": 10.72044, + "49": 10.73376, + "50": 10.71461 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 23026956.0, - "2": 22914276.0, - "3": 22772858.0, - "4": 22840912.0, - "5": 22847244.0, - "6": 22809928.0, - "7": 22935484.0, - "8": 22675290.0, - "9": 22821474.0, - "10": 22553496.0, - "11": 22817820.0, - "12": 22705456.0, - "13": 23386588.0, - "14": 23056392.0, - "15": 22782570.0, - "16": 22886552.0, - "17": 22993644.0, - "18": 23059720.0, - "19": 23149352.0, - "20": 22786660.0, - "21": 22982180.0, - "22": 23009416.0, - "23": 22691960.0, - "24": 22926520.0, - "25": 22700328.0, - "26": 23067802.0, - "27": 22865720.0, - "28": 23062988.0, - "29": 23042732.0, - "30": 23007980.0, - "31": 22971798.0, - "32": 22728988.0, - "33": 22803576.0, - "34": 23139474.0, - "35": 22814012.0, - "36": 22758668.0, - "37": 23167024.0, - "38": 23028582.0, - "39": 23049086.0, - "40": 22813352.0, - "41": 23128888.0, - "42": 22752544.0, - "43": 23050854.0, - "44": 22768246.0, - "45": 22910828.0, - "46": 22797188.0, - "47": 22912552.0, - "48": 22896780.0, - "49": 22951704.0, - "50": 22708438.0 + "1": 23026914.0, + "2": 22914288.0, + "3": 22772932.0, + "4": 22840940.0, + "5": 22847236.0, + "6": 22809846.0, + "7": 22935454.0, + "8": 22675362.0, + "9": 22821508.0, + "10": 22553590.0, + "11": 22817774.0, + "12": 22705564.0, + "13": 23386640.0, + "14": 23056384.0, + "15": 22782556.0, + "16": 22886520.0, + "17": 22993670.0, + "18": 23059684.0, + "19": 23149354.0, + "20": 22786608.0, + "21": 22982160.0, + "22": 23009406.0, + "23": 22691992.0, + "24": 22926500.0, + "25": 22700388.0, + "26": 23067870.0, + "27": 22865636.0, + "28": 23062934.0, + "29": 23042774.0, + "30": 23008000.0, + "31": 22971784.0, + "32": 22728968.0, + "33": 22803548.0, + "34": 23139398.0, + "35": 22813978.0, + "36": 22758748.0, + "37": 23167036.0, + "38": 23028472.0, + "39": 23049044.0, + "40": 22813320.0, + "41": 23128928.0, + "42": 22752548.0, + "43": 23050880.0, + "44": 22768264.0, + "45": 22910844.0, + "46": 22797140.0, + "47": 22912504.0, + "48": 22896768.0, + "49": 22951740.0, + "50": 22708450.0 } }, "mem-allocated-bytes": { @@ -175,7 +175,7 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1122385408.0, + "1": 1123433984.0, "2": 1245635072.0, "3": 1245635072.0, "4": 1245635072.0, @@ -200,31 +200,31 @@ "23": 1245635072.0, "24": 1245635072.0, "25": 1245635072.0, - "26": 1245635072.0, - "27": 1245635072.0, - "28": 1245635072.0, - "29": 1245635072.0, - "30": 1245635072.0, - "31": 1245635072.0, - "32": 1245635072.0, - "33": 1245635072.0, - "34": 1245635072.0, - "35": 1245635072.0, - "36": 1245635072.0, - "37": 1245635072.0, - "38": 1245635072.0, - "39": 1245635072.0, - "40": 1245635072.0, - "41": 1245635072.0, - "42": 1245635072.0, - "43": 1245635072.0, - "44": 1245635072.0, - "45": 1245635072.0, - "46": 1245635072.0, - "47": 1245635072.0, - "48": 1245635072.0, - "49": 1245635072.0, - "50": 1245635072.0 + "26": 1248255488.0, + "27": 1248255488.0, + "28": 1248255488.0, + "29": 1248255488.0, + "30": 1248255488.0, + "31": 1248255488.0, + "32": 1248255488.0, + "33": 1248255488.0, + "34": 1248255488.0, + "35": 1248255488.0, + "36": 1248255488.0, + "37": 1248255488.0, + "38": 1248255488.0, + "39": 1248255488.0, + "40": 1248255488.0, + "41": 1248255488.0, + "42": 1248255488.0, + "43": 1248255488.0, + "44": 1248255488.0, + "45": 1248255488.0, + "46": 1248255488.0, + "47": 1248255488.0, + "48": 1248255488.0, + "49": 1248255488.0, + "50": 1248255488.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.85803, - "3": 0.16855, - "4": 0.15096, - "5": 0.14968, - "6": 0.15038, - "7": 0.14966, - "8": 0.15095, - "9": 0.15123, - "10": 0.15122, - "11": 0.15003, - "12": 0.15015, - "13": 0.15926, - "14": 0.15178, - "15": 0.15011, - "16": 0.15343, - "17": 0.15085, - "18": 0.15258, - "19": 0.15062, - "20": 0.15269, - "21": 0.15054, - "22": 0.15301, - "23": 0.15094, - "24": 0.15254, - "25": 0.15092, - "26": 0.24599, - "27": 0.19144, - "28": 0.18705, - "29": 0.15242, - "30": 0.15076, - "31": 0.15282, - "32": 0.15101, - "33": 0.15252, - "34": 0.15123, - "35": 0.15223, - "36": 0.15139, - "37": 0.15305, - "38": 0.15167, - "39": 0.15316, - "40": 0.15019, - "41": 0.15171, - "42": 0.15065, - "43": 0.15302, - "44": 0.15076, - "45": 0.15277, - "46": 0.15062, - "47": 0.15247, - "48": 0.15036, - "49": 0.15362, - "50": 0.15185 + "2": 7.38558, + "3": 0.17642, + "4": 0.13152, + "5": 0.12879, + "6": 0.12737, + "7": 0.12745, + "8": 0.12999, + "9": 0.13005, + "10": 0.13027, + "11": 0.12891, + "12": 0.12945, + "13": 0.13192, + "14": 0.13042, + "15": 0.13107, + "16": 0.13013, + "17": 0.13072, + "18": 0.13058, + "19": 0.1288, + "20": 0.12858, + "21": 0.1297, + "22": 0.13264, + "23": 0.13244, + "24": 0.13488, + "25": 0.13214, + "26": 0.23594, + "27": 0.15056, + "28": 0.17116, + "29": 0.13573, + "30": 0.1945, + "31": 0.12978, + "32": 0.13149, + "33": 0.1296, + "34": 0.13062, + "35": 0.12906, + "36": 0.13202, + "37": 0.13297, + "38": 0.21317, + "39": 0.13, + "40": 0.13201, + "41": 0.13349, + "42": 0.13265, + "43": 0.1336, + "44": 0.13433, + "45": 0.13075, + "46": 0.13234, + "47": 0.22724, + "48": 0.13118, + "49": 0.13257, + "50": 0.13139 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json index 6d3a37e6e1a..b995bb68700 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92858, - "2": 10.92691, - "3": 10.9228, - "4": 10.92385, - "5": 10.92018, + "1": 10.92859, + "2": 10.9269, + "3": 10.92279, + "4": 10.92378, + "5": 10.92022, "6": 10.92481, - "7": 10.91599, - "8": 10.92353, - "9": 10.91957, - "10": 10.92508, - "11": 10.92005, - "12": 10.92356, - "13": 10.91465, - "14": 10.91785, - "15": 10.91656, - "16": 10.92311, - "17": 10.91335, - "18": 10.91241, - "19": 10.90858, - "20": 10.90755, - "21": 10.89514, - "22": 10.89723, - "23": 10.89723, - "24": 10.89583, - "25": 10.90396, - "26": 10.89488, - "27": 10.88792, - "28": 10.86575, - "29": 10.85896, - "30": 10.85707, - "31": 10.84766, - "32": 10.86079, - "33": 10.84972, - "34": 10.83674, - "35": 10.84444, - "36": 10.83032, - "37": 10.82693, - "38": 10.80413, + "7": 10.91589, + "8": 10.92356, + "9": 10.91952, + "10": 10.92509, + "11": 10.92003, + "12": 10.92357, + "13": 10.9147, + "14": 10.91789, + "15": 10.91662, + "16": 10.92306, + "17": 10.91332, + "18": 10.9124, + "19": 10.90854, + "20": 10.9076, + "21": 10.89511, + "22": 10.89724, + "23": 10.89727, + "24": 10.89578, + "25": 10.90401, + "26": 10.89489, + "27": 10.88789, + "28": 10.86573, + "29": 10.85901, + "30": 10.85709, + "31": 10.8477, + "32": 10.86082, + "33": 10.84971, + "34": 10.83681, + "35": 10.84447, + "36": 10.83037, + "37": 10.82695, + "38": 10.80411, "39": 10.77934, - "40": 10.77162, - "41": 10.75997, - "42": 10.75927, + "40": 10.77166, + "41": 10.76003, + "42": 10.75933, "43": 10.75404, - "44": 10.74361, - "45": 10.73916, - "46": 10.71446, - "47": 10.71009, - "48": 10.68542, - "49": 10.69158, - "50": 10.6792 + "44": 10.74359, + "45": 10.73922, + "46": 10.71445, + "47": 10.71014, + "48": 10.68548, + "49": 10.6915, + "50": 10.67923 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 23025846.0, - "2": 22914698.0, - "3": 22773682.0, - "4": 22842060.0, - "5": 22847304.0, - "6": 22810108.0, - "7": 22935284.0, - "8": 22675112.0, - "9": 22822952.0, - "10": 22552496.0, - "11": 22818072.0, - "12": 22705452.0, - "13": 23386520.0, - "14": 23055528.0, - "15": 22782176.0, - "16": 22886556.0, - "17": 22992588.0, + "1": 23025860.0, + "2": 22914720.0, + "3": 22773588.0, + "4": 22842072.0, + "5": 22847252.0, + "6": 22810068.0, + "7": 22935268.0, + "8": 22675124.0, + "9": 22822992.0, + "10": 22552360.0, + "11": 22817968.0, + "12": 22705544.0, + "13": 23386596.0, + "14": 23055506.0, + "15": 22782216.0, + "16": 22886524.0, + "17": 22992648.0, "18": 23059260.0, - "19": 23148592.0, - "20": 22786778.0, - "21": 22982468.0, - "22": 23009396.0, - "23": 22693052.0, - "24": 22926420.0, - "25": 22699896.0, - "26": 23068178.0, - "27": 22865288.0, - "28": 23063142.0, - "29": 23043304.0, - "30": 23007890.0, - "31": 22972192.0, - "32": 22730568.0, - "33": 22804668.0, - "34": 23140012.0, - "35": 22814154.0, - "36": 22759064.0, - "37": 23167088.0, - "38": 23029028.0, - "39": 23048668.0, - "40": 22813504.0, - "41": 23128932.0, - "42": 22752812.0, - "43": 23050544.0, - "44": 22768146.0, - "45": 22910452.0, - "46": 22797168.0, - "47": 22912692.0, - "48": 22897264.0, - "49": 22951794.0, - "50": 22707566.0 + "19": 23148520.0, + "20": 22786856.0, + "21": 22982420.0, + "22": 23009412.0, + "23": 22693004.0, + "24": 22926362.0, + "25": 22699796.0, + "26": 23068216.0, + "27": 22865320.0, + "28": 23063088.0, + "29": 23043308.0, + "30": 23007930.0, + "31": 22972168.0, + "32": 22730592.0, + "33": 22804614.0, + "34": 23140082.0, + "35": 22814240.0, + "36": 22758996.0, + "37": 23167084.0, + "38": 23029010.0, + "39": 23048660.0, + "40": 22813540.0, + "41": 23128888.0, + "42": 22752764.0, + "43": 23050580.0, + "44": 22768264.0, + "45": 22910384.0, + "46": 22797176.0, + "47": 22912748.0, + "48": 22897280.0, + "49": 22951760.0, + "50": 22707564.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.04488, - "3": 0.14123, - "4": 0.13989, - "5": 0.13964, - "6": 0.14307, - "7": 0.14186, - "8": 0.13867, - "9": 0.13845, - "10": 0.13944, - "11": 0.13857, - "12": 0.14012, - "13": 0.14088, - "14": 0.13898, - "15": 0.13896, - "16": 0.13994, - "17": 0.13894, - "18": 0.13867, - "19": 0.13866, - "20": 0.14137, - "21": 0.13893, - "22": 0.13714, - "23": 0.14007, - "24": 0.13871, - "25": 0.14434, - "26": 0.17261, - "27": 0.16786, - "28": 0.14429, - "29": 0.14179, - "30": 0.14227, - "31": 0.14275, - "32": 0.14195, - "33": 0.1392, - "34": 0.13898, - "35": 0.13863, - "36": 0.14135, - "37": 0.13876, - "38": 0.13785, - "39": 0.1478, - "40": 0.14002, - "41": 0.13994, - "42": 0.14007, - "43": 0.13829, - "44": 0.13827, - "45": 0.13753, - "46": 0.13849, - "47": 0.14184, - "48": 0.13854, - "49": 0.13966, - "50": 0.13944 + "2": 5.58724, + "3": 0.14313, + "4": 0.14354, + "5": 0.14005, + "6": 0.14403, + "7": 0.13973, + "8": 0.13987, + "9": 0.13952, + "10": 0.14117, + "11": 0.14051, + "12": 0.13999, + "13": 0.14113, + "14": 0.13846, + "15": 0.13843, + "16": 0.14276, + "17": 0.13878, + "18": 0.14122, + "19": 0.13947, + "20": 0.13947, + "21": 0.13892, + "22": 0.14081, + "23": 0.14012, + "24": 0.14054, + "25": 0.14178, + "26": 0.17837, + "27": 0.1743, + "28": 0.15624, + "29": 0.14965, + "30": 0.14711, + "31": 0.14417, + "32": 0.14308, + "33": 0.14761, + "34": 0.14701, + "35": 0.14557, + "36": 0.1453, + "37": 0.14438, + "38": 0.14226, + "39": 0.14677, + "40": 0.14531, + "41": 0.14612, + "42": 0.14142, + "43": 0.14779, + "44": 0.1431, + "45": 0.14468, + "46": 0.14237, + "47": 0.14248, + "48": 0.1421, + "49": 0.14234, + "50": 0.1419 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json index 9fd43415bf0..70096b81e8a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.95497, - "2": 10.95336, - "3": 10.95084, - "4": 10.95138, - "5": 10.95095, - "6": 10.95418, - "7": 10.95215, - "8": 10.95329, - "9": 10.9475, + "1": 10.95495, + "2": 10.95329, + "3": 10.95082, + "4": 10.95133, + "5": 10.9509, + "6": 10.95414, + "7": 10.95218, + "8": 10.95334, + "9": 10.94752, "10": 10.94585, - "11": 10.94813, - "12": 10.96094, - "13": 10.9515, - "14": 10.95222, - "15": 10.94823, - "16": 10.94694, - "17": 10.95226, - "18": 10.94757, - "19": 10.95164, - "20": 10.93972, - "21": 10.93985, - "22": 10.93777, + "11": 10.94815, + "12": 10.96099, + "13": 10.95154, + "14": 10.95219, + "15": 10.94822, + "16": 10.94693, + "17": 10.95232, + "18": 10.94756, + "19": 10.95156, + "20": 10.93971, + "21": 10.93986, + "22": 10.93779, "23": 10.92832, - "24": 10.92783, - "25": 10.93223, - "26": 10.92799, - "27": 10.9229, - "28": 10.89388, - "29": 10.88953, - "30": 10.8825, - "31": 10.88534, - "32": 10.88053, - "33": 10.87143, - "34": 10.86879, - "35": 10.86909, + "24": 10.92785, + "25": 10.93225, + "26": 10.92789, + "27": 10.92298, + "28": 10.89386, + "29": 10.88954, + "30": 10.88251, + "31": 10.88536, + "32": 10.88054, + "33": 10.87139, + "34": 10.86882, + "35": 10.86911, "36": 10.86039, - "37": 10.85941, - "38": 10.83806, + "37": 10.85944, + "38": 10.83809, "39": 10.80305, "40": 10.80807, "41": 10.79481, - "42": 10.77754, - "43": 10.77432, - "44": 10.77182, - "45": 10.77167, - "46": 10.75137, - "47": 10.73716, - "48": 10.71898, + "42": 10.77756, + "43": 10.77435, + "44": 10.77189, + "45": 10.77173, + "46": 10.75143, + "47": 10.73718, + "48": 10.71907, "49": 10.73121, - "50": 10.71218 + "50": 10.71224 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 23026300.0, - "2": 22914086.0, - "3": 22773382.0, - "4": 22840872.0, - "5": 22848084.0, - "6": 22810006.0, - "7": 22935596.0, - "8": 22675152.0, - "9": 22822660.0, - "10": 22552656.0, - "11": 22817710.0, - "12": 22705288.0, - "13": 23386302.0, - "14": 23056148.0, - "15": 22781850.0, - "16": 22886906.0, - "17": 22993636.0, - "18": 23058726.0, - "19": 23148372.0, - "20": 22787076.0, - "21": 22981484.0, - "22": 23010492.0, - "23": 22692514.0, - "24": 22925772.0, - "25": 22699682.0, - "26": 23068768.0, - "27": 22865260.0, - "28": 23063292.0, - "29": 23042696.0, - "30": 23008720.0, - "31": 22972018.0, - "32": 22730204.0, - "33": 22804204.0, - "34": 23139172.0, - "35": 22813746.0, - "36": 22758552.0, - "37": 23167014.0, - "38": 23028354.0, - "39": 23048620.0, - "40": 22812828.0, - "41": 23129236.0, - "42": 22752788.0, - "43": 23050796.0, - "44": 22767704.0, - "45": 22909896.0, - "46": 22797186.0, - "47": 22913150.0, - "48": 22896154.0, - "49": 22952004.0, - "50": 22707804.0 + "1": 23026284.0, + "2": 22914140.0, + "3": 22773236.0, + "4": 22840898.0, + "5": 22848020.0, + "6": 22809990.0, + "7": 22935564.0, + "8": 22675156.0, + "9": 22822612.0, + "10": 22552600.0, + "11": 22817702.0, + "12": 22705280.0, + "13": 23386330.0, + "14": 23056232.0, + "15": 22781852.0, + "16": 22886936.0, + "17": 22993724.0, + "18": 23058680.0, + "19": 23148428.0, + "20": 22787088.0, + "21": 22981476.0, + "22": 23010380.0, + "23": 22692468.0, + "24": 22925756.0, + "25": 22699584.0, + "26": 23068772.0, + "27": 22865264.0, + "28": 23063288.0, + "29": 23042648.0, + "30": 23008806.0, + "31": 22972060.0, + "32": 22730272.0, + "33": 22804248.0, + "34": 23139150.0, + "35": 22813676.0, + "36": 22758546.0, + "37": 23167030.0, + "38": 23028400.0, + "39": 23048512.0, + "40": 22812806.0, + "41": 23129286.0, + "42": 22752824.0, + "43": 23050778.0, + "44": 22767736.0, + "45": 22909854.0, + "46": 22797204.0, + "47": 22913040.0, + "48": 22896188.0, + "49": 22951984.0, + "50": 22707788.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.91356, - "3": 2.24295, - "4": 1.92336, - "5": 1.61239, - "6": 1.86835, - "7": 1.45068, - "8": 2.21581, - "9": 2.65865, - "10": 1.77534, - "11": 2.10301, - "12": 1.69138, - "13": 2.18653, - "14": 2.08534, - "15": 1.69653, - "16": 2.02157, - "17": 2.59276, - "18": 1.82518, - "19": 2.03353, - "20": 1.93249, - "21": 2.19679, - "22": 1.74841, - "23": 1.88576, - "24": 1.77505, - "25": 1.98962, - "26": 1.95857, - "27": 1.83598, - "28": 1.79531, - "29": 1.52829, - "30": 1.83218, - "31": 1.73951, - "32": 2.62809, - "33": 1.69551, - "34": 1.51829, - "35": 2.27775, - "36": 2.20222, - "37": 1.6045, - "38": 1.54015, - "39": 2.09868, - "40": 1.6369, - "41": 1.61275, - "42": 2.73991, - "43": 1.57157, - "44": 1.88966, - "45": 2.03006, - "46": 1.84791, - "47": 1.72333, - "48": 2.02252, - "49": 1.69002, - "50": 2.47843 + "2": 5.35324, + "3": 1.90259, + "4": 1.81672, + "5": 1.44277, + "6": 1.4421, + "7": 1.64447, + "8": 1.93048, + "9": 2.28103, + "10": 2.02464, + "11": 1.85619, + "12": 1.61223, + "13": 1.56512, + "14": 2.35042, + "15": 1.51625, + "16": 1.55433, + "17": 1.92245, + "18": 1.48284, + "19": 1.78351, + "20": 1.56182, + "21": 1.70962, + "22": 1.40736, + "23": 1.80807, + "24": 1.26426, + "25": 1.92093, + "26": 1.57618, + "27": 1.76219, + "28": 1.44041, + "29": 1.41013, + "30": 1.46576, + "31": 1.27526, + "32": 2.05496, + "33": 1.42442, + "34": 1.40809, + "35": 2.05549, + "36": 1.74733, + "37": 1.69823, + "38": 1.1762, + "39": 1.95014, + "40": 1.81524, + "41": 1.71343, + "42": 1.70282, + "43": 1.59845, + "44": 1.3395, + "45": 1.57432, + "46": 1.43195, + "47": 1.618, + "48": 1.67815, + "49": 1.54797, + "50": 1.89787 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json index 2eca5ad0eec..7eb57562ddd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92942, - "2": 10.92777, - "3": 10.92678, - "4": 10.92616, - "5": 10.9267, - "6": 10.92764, - "7": 10.92398, - "8": 10.91832, - "9": 10.91964, - "10": 10.91772, - "11": 10.9081, - "12": 10.91698, - "13": 10.91063, - "14": 10.90207, - "15": 10.88113, - "16": 10.87325, - "17": 10.86927, + "1": 10.92954, + "2": 10.92779, + "3": 10.92675, + "4": 10.92617, + "5": 10.92685, + "6": 10.92766, + "7": 10.92394, + "8": 10.91823, + "9": 10.91968, + "10": 10.91783, + "11": 10.90805, + "12": 10.91691, + "13": 10.91061, + "14": 10.90208, + "15": 10.88117, + "16": 10.87323, + "17": 10.86924, "18": 10.86347, "19": 10.86027, - "20": 10.7704, - "21": 10.7558, - "22": 10.75377, - "23": 10.74581, + "20": 10.77048, + "21": 10.75588, + "22": 10.75378, + "23": 10.74582, "24": 10.71439, - "25": 10.71349, - "26": 10.7028, - "27": 10.65473, - "28": 10.57862, - "29": 10.55897, - "30": 10.54749, - "31": 10.52972, - "32": 10.51715, - "33": 10.48067, - "34": 10.45115, - "35": 10.45041, - "36": 10.43318, - "37": 10.39352, + "25": 10.71356, + "26": 10.70281, + "27": 10.65478, + "28": 10.57866, + "29": 10.55902, + "30": 10.54751, + "31": 10.52974, + "32": 10.51726, + "33": 10.48074, + "34": 10.45122, + "35": 10.45042, + "36": 10.43325, + "37": 10.39358, "38": 10.39541, - "39": 10.36368, - "40": 10.35672, - "41": 10.32573, - "42": 10.3077, - "43": 10.28179, - "44": 10.25427, - "45": 10.2702, - "46": 10.23518, - "47": 10.21978, - "48": 10.16978, - "49": 10.17822, - "50": 10.17837, - "51": 10.1923, - "52": 10.13727, - "53": 10.13533, - "54": 10.10653, - "55": 10.08096, - "56": 10.11297, - "57": 10.09527, - "58": 10.10881, - "59": 10.05601, - "60": 10.07522, - "61": 10.02733, - "62": 10.00227, - "63": 10.07213, - "64": 10.03491, - "65": 10.00278, - "66": 10.02915, - "67": 10.00666, + "39": 10.36364, + "40": 10.35673, + "41": 10.32578, + "42": 10.30776, + "43": 10.28189, + "44": 10.25432, + "45": 10.27025, + "46": 10.23525, + "47": 10.21977, + "48": 10.1698, + "49": 10.17827, + "50": 10.17844, + "51": 10.19231, + "52": 10.13728, + "53": 10.1354, + "54": 10.10661, + "55": 10.08098, + "56": 10.11295, + "57": 10.09534, + "58": 10.10889, + "59": 10.05609, + "60": 10.07523, + "61": 10.02741, + "62": 10.00226, + "63": 10.07217, + "64": 10.03496, + "65": 10.00276, + "66": 10.02921, + "67": 10.00669, "68": 9.96974, - "69": 9.99041, - "70": 9.97339, - "71": 10.00125, - "72": 9.97761, + "69": 9.99044, + "70": 9.97337, + "71": 10.00127, + "72": 9.97765, "73": 9.97164, - "74": 9.95659, - "75": 9.92901, - "76": 9.96291, - "77": 9.9572, - "78": 9.90596, - "79": 9.90998, - "80": 9.93062, - "81": 9.95611, - "82": 9.89121, - "83": 9.85516, - "84": 9.78771, - "85": 9.78742, - "86": 9.88333, - "87": 9.91031, + "74": 9.95662, + "75": 9.92907, + "76": 9.96297, + "77": 9.95726, + "78": 9.90604, + "79": 9.91006, + "80": 9.93069, + "81": 9.95613, + "82": 9.89127, + "83": 9.85522, + "84": 9.78775, + "85": 9.78743, + "86": 9.88339, + "87": 9.91033, "88": 9.88594, - "89": 9.81829, - "90": 9.8142, - "91": 9.82558, - "92": 9.81425, - "93": 9.75216, + "89": 9.81827, + "90": 9.81423, + "91": 9.82562, + "92": 9.81426, + "93": 9.75215, "94": 9.83216, - "95": 9.82474, - "96": 9.80679, + "95": 9.82478, + "96": 9.80682, "97": 9.74328, - "98": 9.77878, - "99": 9.81992, - "100": 9.71216 + "98": 9.77881, + "99": 9.81997, + "100": 9.71224 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1838.0, - "2": 1679.0, - "3": 1823.0, - "4": 1657.0, - "5": 1659.0, - "6": 1659.0, - "7": 1901.0, - "8": 1778.0, - "9": 1713.0, - "10": 1665.0, - "11": 1674.0, - "12": 1579.0, - "13": 1665.0, - "14": 1874.0, - "15": 1587.0, - "16": 1603.0, - "17": 1719.0, - "18": 1724.0, - "19": 1650.0, - "20": 1639.0, - "21": 1743.0, - "22": 1788.0, - "23": 1639.0, - "24": 1761.0, - "25": 1703.0, - "26": 1754.0, - "27": 1780.0, - "28": 1879.0, - "29": 1887.0, - "30": 1815.0, - "31": 1949.0, - "32": 1992.0, - "33": 1933.0, - "34": 1838.0, - "35": 1997.0, - "36": 1945.0, - "37": 2203.0, - "38": 2064.0, - "39": 2141.0, - "40": 2254.0, - "41": 2283.0, - "42": 2037.0, - "43": 2343.0, - "44": 2171.0, - "45": 2487.0, - "46": 2526.0, - "47": 2456.0, - "48": 2611.0, - "49": 2921.0, - "50": 2600.0, - "51": 2510.0, - "52": 2852.0, - "53": 2598.0, - "54": 2774.0, - "55": 2609.0, - "56": 2730.0, - "57": 2196.0, - "58": 3537.0, - "59": 2868.0, - "60": 3030.0, - "61": 2835.0, - "62": 3206.0, - "63": 3374.0, - "64": 3656.0, - "65": 2646.0, - "66": 2944.0, - "67": 3862.0, - "68": 3436.0, - "69": 3030.0, - "70": 3271.0, - "71": 3210.0, - "72": 3057.0, - "73": 3453.0, - "74": 3277.0, - "75": 3257.0, - "76": 3250.0, - "77": 3639.0, - "78": 3207.0, - "79": 3079.0, - "80": 3041.0, - "81": 3578.0, - "82": 2879.0, - "83": 3166.0, - "84": 2924.0, - "85": 2537.0, - "86": 3013.0, - "87": 2973.0, - "88": 3029.0, - "89": 2997.0, - "90": 3837.0, - "91": 3034.0, - "92": 2813.0, - "93": 3084.0, - "94": 2992.0, - "95": 3249.0, - "96": 3345.0, - "97": 3465.0, - "98": 3255.0, - "99": 3305.0, - "100": 3279.0 + "1": 1798.0, + "2": 1730.0, + "3": 1779.0, + "4": 1679.0, + "5": 1748.0, + "6": 1640.0, + "7": 1794.0, + "8": 1728.0, + "9": 1693.0, + "10": 1809.0, + "11": 1614.0, + "12": 1639.0, + "13": 1847.0, + "14": 1794.0, + "15": 1715.0, + "16": 1667.0, + "17": 1613.0, + "18": 1709.0, + "19": 1627.0, + "20": 1653.0, + "21": 1719.0, + "22": 1663.0, + "23": 1603.0, + "24": 1772.0, + "25": 1718.0, + "26": 1739.0, + "27": 1804.0, + "28": 1814.0, + "29": 1882.0, + "30": 1745.0, + "31": 1942.0, + "32": 1894.0, + "33": 1996.0, + "34": 1897.0, + "35": 2072.0, + "36": 1985.0, + "37": 2210.0, + "38": 2076.0, + "39": 2251.0, + "40": 2205.0, + "41": 2262.0, + "42": 1978.0, + "43": 2386.0, + "44": 2219.0, + "45": 2468.0, + "46": 2374.0, + "47": 2386.0, + "48": 2744.0, + "49": 2913.0, + "50": 2603.0, + "51": 2527.0, + "52": 2843.0, + "53": 2670.0, + "54": 2854.0, + "55": 2552.0, + "56": 2726.0, + "57": 2248.0, + "58": 3587.0, + "59": 2898.0, + "60": 3025.0, + "61": 2849.0, + "62": 3243.0, + "63": 3421.0, + "64": 3474.0, + "65": 2682.0, + "66": 2921.0, + "67": 3813.0, + "68": 3404.0, + "69": 3094.0, + "70": 3315.0, + "71": 3073.0, + "72": 2999.0, + "73": 3614.0, + "74": 3364.0, + "75": 3332.0, + "76": 3326.0, + "77": 3721.0, + "78": 3282.0, + "79": 3191.0, + "80": 2947.0, + "81": 3677.0, + "82": 2893.0, + "83": 3141.0, + "84": 2992.0, + "85": 2540.0, + "86": 3014.0, + "87": 2909.0, + "88": 2909.0, + "89": 3095.0, + "90": 3741.0, + "91": 2950.0, + "92": 2856.0, + "93": 3015.0, + "94": 3098.0, + "95": 3318.0, + "96": 3395.0, + "97": 3654.0, + "98": 3093.0, + "99": 3350.0, + "100": 3189.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 514359808.0, - "2": 514359808.0, - "3": 514359808.0, - "4": 514359808.0, - "5": 514359808.0, - "6": 514359808.0, - "7": 514359808.0, - "8": 514359808.0, - "9": 514359808.0, - "10": 514359808.0, - "11": 514359808.0, - "12": 514359808.0, - "13": 514359808.0, - "14": 514359808.0, - "15": 514359808.0, - "16": 514359808.0, - "17": 514359808.0, - "18": 514359808.0, - "19": 514359808.0, - "20": 514359808.0, - "21": 514359808.0, - "22": 514359808.0, - "23": 514359808.0, - "24": 514359808.0, - "25": 514359808.0, - "26": 514359808.0, - "27": 514359808.0, - "28": 514359808.0, - "29": 514359808.0, - "30": 514359808.0, - "31": 514359808.0, - "32": 514359808.0, - "33": 514359808.0, - "34": 514359808.0, - "35": 514359808.0, - "36": 514359808.0, - "37": 514359808.0, - "38": 514359808.0, - "39": 514359808.0, - "40": 514359808.0, - "41": 514359808.0, - "42": 514359808.0, - "43": 514359808.0, - "44": 514359808.0, - "45": 514359808.0, - "46": 514359808.0, - "47": 514359808.0, - "48": 514359808.0, - "49": 514359808.0, - "50": 514359808.0, - "51": 514359808.0, - "52": 514359808.0, - "53": 514359808.0, - "54": 514359808.0, - "55": 514359808.0, - "56": 514359808.0, - "57": 514359808.0, - "58": 514359808.0, - "59": 514359808.0, - "60": 514359808.0, - "61": 514359808.0, - "62": 514359808.0, - "63": 514359808.0, - "64": 514359808.0, - "65": 514359808.0, - "66": 514359808.0, - "67": 514359808.0, - "68": 514359808.0, - "69": 514359808.0, - "70": 514359808.0, - "71": 514359808.0, - "72": 514359808.0, - "73": 514359808.0, - "74": 514359808.0, - "75": 514359808.0, - "76": 514359808.0, - "77": 514359808.0, - "78": 514359808.0, - "79": 514359808.0, - "80": 514359808.0, - "81": 514359808.0, - "82": 514359808.0, - "83": 514359808.0, - "84": 514359808.0, - "85": 514359808.0, - "86": 514359808.0, - "87": 514359808.0, - "88": 514359808.0, - "89": 514359808.0, - "90": 514359808.0, - "91": 514359808.0, - "92": 514359808.0, - "93": 514359808.0, - "94": 514359808.0, - "95": 514359808.0, - "96": 514359808.0, - "97": 514359808.0, - "98": 514359808.0, - "99": 514359808.0, - "100": 514359808.0 + "1": 514884096.0, + "2": 514884096.0, + "3": 514884096.0, + "4": 514884096.0, + "5": 514884096.0, + "6": 514884096.0, + "7": 514884096.0, + "8": 514884096.0, + "9": 514884096.0, + "10": 514884096.0, + "11": 514884096.0, + "12": 514884096.0, + "13": 514884096.0, + "14": 514884096.0, + "15": 514884096.0, + "16": 514884096.0, + "17": 514884096.0, + "18": 514884096.0, + "19": 514884096.0, + "20": 514884096.0, + "21": 514884096.0, + "22": 514884096.0, + "23": 514884096.0, + "24": 514884096.0, + "25": 514884096.0, + "26": 514884096.0, + "27": 514884096.0, + "28": 514884096.0, + "29": 514884096.0, + "30": 514884096.0, + "31": 514884096.0, + "32": 514884096.0, + "33": 514884096.0, + "34": 514884096.0, + "35": 514884096.0, + "36": 514884096.0, + "37": 514884096.0, + "38": 514884096.0, + "39": 514884096.0, + "40": 514884096.0, + "41": 514884096.0, + "42": 514884096.0, + "43": 514884096.0, + "44": 514884096.0, + "45": 514884096.0, + "46": 514884096.0, + "47": 514884096.0, + "48": 514884096.0, + "49": 514884096.0, + "50": 514884096.0, + "51": 514884096.0, + "52": 514884096.0, + "53": 514884096.0, + "54": 514884096.0, + "55": 514884096.0, + "56": 514884096.0, + "57": 514884096.0, + "58": 514884096.0, + "59": 514884096.0, + "60": 514884096.0, + "61": 514884096.0, + "62": 514884096.0, + "63": 514884096.0, + "64": 514884096.0, + "65": 514884096.0, + "66": 514884096.0, + "67": 514884096.0, + "68": 514884096.0, + "69": 514884096.0, + "70": 514884096.0, + "71": 514884096.0, + "72": 514884096.0, + "73": 514884096.0, + "74": 514884096.0, + "75": 514884096.0, + "76": 514884096.0, + "77": 514884096.0, + "78": 514884096.0, + "79": 514884096.0, + "80": 514884096.0, + "81": 514884096.0, + "82": 514884096.0, + "83": 514884096.0, + "84": 514884096.0, + "85": 514884096.0, + "86": 514884096.0, + "87": 514884096.0, + "88": 514884096.0, + "89": 514884096.0, + "90": 514884096.0, + "91": 514884096.0, + "92": 514884096.0, + "93": 514884096.0, + "94": 514884096.0, + "95": 514884096.0, + "96": 514884096.0, + "97": 514884096.0, + "98": 514884096.0, + "99": 514884096.0, + "100": 514884096.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1258060288.0, - "2": 1437084160.0, - "3": 1437084160.0, - "4": 1437084160.0, - "5": 1437084160.0, - "6": 1437084160.0, - "7": 1437084160.0, - "8": 1437084160.0, - "9": 1437084160.0, - "10": 1437084160.0, - "11": 1437084160.0, - "12": 1437084160.0, - "13": 1437084160.0, - "14": 1437084160.0, - "15": 1437084160.0, - "16": 1437084160.0, - "17": 1437084160.0, - "18": 1437084160.0, - "19": 1437084160.0, - "20": 1437084160.0, - "21": 1437084160.0, - "22": 1437084160.0, - "23": 1437084160.0, - "24": 1437084160.0, - "25": 1437084160.0, - "26": 1437084160.0, - "27": 1437084160.0, - "28": 1437084160.0, - "29": 1437084160.0, - "30": 1437084160.0, - "31": 1437084160.0, - "32": 1437084160.0, - "33": 1437084160.0, - "34": 1437084160.0, - "35": 1437084160.0, - "36": 1437084160.0, - "37": 1437084160.0, - "38": 1437084160.0, - "39": 1437084160.0, - "40": 1437084160.0, - "41": 1437084160.0, - "42": 1437084160.0, - "43": 1437084160.0, - "44": 1437084160.0, - "45": 1437084160.0, - "46": 1437084160.0, - "47": 1437084160.0, - "48": 1437084160.0, - "49": 1437084160.0, - "50": 1437084160.0, - "51": 1437084160.0, - "52": 1437084160.0, - "53": 1437084160.0, - "54": 1437084160.0, - "55": 1437084160.0, - "56": 1437084160.0, - "57": 1437084160.0, - "58": 1437084160.0, - "59": 1437084160.0, - "60": 1437084160.0, - "61": 1437084160.0, - "62": 1437084160.0, - "63": 1437084160.0, - "64": 1437084160.0, - "65": 1437084160.0, - "66": 1437084160.0, - "67": 1437084160.0, - "68": 1437084160.0, - "69": 1437084160.0, - "70": 1437084160.0, - "71": 1437084160.0, - "72": 1437084160.0, - "73": 1437084160.0, - "74": 1437084160.0, - "75": 1437084160.0, - "76": 1437084160.0, - "77": 1437084160.0, - "78": 1437084160.0, - "79": 1437084160.0, - "80": 1437084160.0, - "81": 1437084160.0, - "82": 1437084160.0, - "83": 1437084160.0, - "84": 1437084160.0, - "85": 1437084160.0, - "86": 1437084160.0, - "87": 1437084160.0, - "88": 1437084160.0, - "89": 1437084160.0, - "90": 1437084160.0, - "91": 1437084160.0, - "92": 1437084160.0, - "93": 1437084160.0, - "94": 1437084160.0, - "95": 1437084160.0, - "96": 1437084160.0, - "97": 1437084160.0, - "98": 1437084160.0, - "99": 1437084160.0, - "100": 1437084160.0 + "1": 1259108864.0, + "2": 1438394880.0, + "3": 1438394880.0, + "4": 1438394880.0, + "5": 1438394880.0, + "6": 1438394880.0, + "7": 1438394880.0, + "8": 1438394880.0, + "9": 1438394880.0, + "10": 1438394880.0, + "11": 1438394880.0, + "12": 1438394880.0, + "13": 1438394880.0, + "14": 1438394880.0, + "15": 1438394880.0, + "16": 1438394880.0, + "17": 1438394880.0, + "18": 1438394880.0, + "19": 1438394880.0, + "20": 1438394880.0, + "21": 1438394880.0, + "22": 1438394880.0, + "23": 1438394880.0, + "24": 1438394880.0, + "25": 1438394880.0, + "26": 1438394880.0, + "27": 1438394880.0, + "28": 1438394880.0, + "29": 1438394880.0, + "30": 1438394880.0, + "31": 1438394880.0, + "32": 1438394880.0, + "33": 1438394880.0, + "34": 1438394880.0, + "35": 1438394880.0, + "36": 1438394880.0, + "37": 1438394880.0, + "38": 1438394880.0, + "39": 1438394880.0, + "40": 1438394880.0, + "41": 1438394880.0, + "42": 1438394880.0, + "43": 1438394880.0, + "44": 1438394880.0, + "45": 1438394880.0, + "46": 1438394880.0, + "47": 1438394880.0, + "48": 1438394880.0, + "49": 1438394880.0, + "50": 1438394880.0, + "51": 1439181312.0, + "52": 1439181312.0, + "53": 1439181312.0, + "54": 1439181312.0, + "55": 1439181312.0, + "56": 1439181312.0, + "57": 1439181312.0, + "58": 1439181312.0, + "59": 1439181312.0, + "60": 1439181312.0, + "61": 1439181312.0, + "62": 1439181312.0, + "63": 1439181312.0, + "64": 1439181312.0, + "65": 1439181312.0, + "66": 1439181312.0, + "67": 1439181312.0, + "68": 1439181312.0, + "69": 1439181312.0, + "70": 1439181312.0, + "71": 1439181312.0, + "72": 1439181312.0, + "73": 1439181312.0, + "74": 1439181312.0, + "75": 1439181312.0, + "76": 1439181312.0, + "77": 1439181312.0, + "78": 1439181312.0, + "79": 1439181312.0, + "80": 1439181312.0, + "81": 1439181312.0, + "82": 1439181312.0, + "83": 1439181312.0, + "84": 1439181312.0, + "85": 1439181312.0, + "86": 1439181312.0, + "87": 1439181312.0, + "88": 1439181312.0, + "89": 1439181312.0, + "90": 1439181312.0, + "91": 1439181312.0, + "92": 1439181312.0, + "93": 1439181312.0, + "94": 1439181312.0, + "95": 1439181312.0, + "96": 1439181312.0, + "97": 1439181312.0, + "98": 1439181312.0, + "99": 1439181312.0, + "100": 1439181312.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.33614, - "3": 0.21859, - "4": 0.20401, - "5": 0.20294, - "6": 0.20287, - "7": 0.20323, - "8": 0.19973, - "9": 0.20956, - "10": 0.20466, - "11": 0.20367, - "12": 0.2028, - "13": 0.20233, - "14": 0.20044, - "15": 0.20097, - "16": 0.20203, - "17": 0.20093, - "18": 0.19986, - "19": 0.1986, - "20": 0.19903, - "21": 0.20277, - "22": 0.20055, - "23": 0.20104, - "24": 0.20056, - "25": 0.20296, - "26": 0.20362, - "27": 0.20317, - "28": 0.20469, - "29": 0.20495, - "30": 0.20509, - "31": 0.20317, - "32": 0.2036, - "33": 0.20396, - "34": 0.20418, - "35": 0.20477, - "36": 0.20485, - "37": 0.20316, - "38": 0.20237, - "39": 0.20152, - "40": 0.20209, - "41": 0.20175, - "42": 0.2015, - "43": 0.20279, - "44": 0.20299, - "45": 0.20191, - "46": 0.20491, - "47": 0.20293, - "48": 0.20144, - "49": 0.20007, - "50": 0.20478, - "51": 0.32377, - "52": 0.25649, - "53": 0.20439, - "54": 0.20127, - "55": 0.20091, - "56": 0.20189, - "57": 0.20068, - "58": 0.20229, - "59": 0.20721, - "60": 0.20166, - "61": 0.20124, - "62": 0.20074, - "63": 0.20072, - "64": 0.20045, - "65": 0.20005, - "66": 0.20051, - "67": 0.20099, - "68": 0.2008, - "69": 0.20085, - "70": 0.20081, - "71": 0.20095, - "72": 0.20138, - "73": 0.20182, - "74": 0.20289, - "75": 0.20198, - "76": 0.19987, - "77": 0.20411, - "78": 0.20491, - "79": 0.20307, - "80": 0.20301, - "81": 0.20498, - "82": 0.20446, - "83": 0.20491, - "84": 0.20515, - "85": 0.20319, - "86": 0.20464, - "87": 0.20603, - "88": 0.20498, - "89": 0.2062, - "90": 0.20467, - "91": 0.2005, - "92": 0.20179, - "93": 0.20188, - "94": 0.20137, - "95": 0.20149, - "96": 0.20349, - "97": 0.20482, - "98": 0.20388, - "99": 0.20426, - "100": 0.20203 + "2": 6.91111, + "3": 0.24358, + "4": 0.1729, + "5": 0.1766, + "6": 0.17634, + "7": 0.17255, + "8": 0.17313, + "9": 0.17254, + "10": 0.17537, + "11": 0.17292, + "12": 0.17179, + "13": 0.17321, + "14": 0.17688, + "15": 0.17322, + "16": 0.17468, + "17": 0.17344, + "18": 0.17555, + "19": 0.17283, + "20": 0.17665, + "21": 0.17528, + "22": 0.17628, + "23": 0.17503, + "24": 0.17686, + "25": 0.17362, + "26": 0.17311, + "27": 0.17527, + "28": 0.17727, + "29": 0.1753, + "30": 0.17605, + "31": 0.17443, + "32": 0.17458, + "33": 0.17569, + "34": 0.17367, + "35": 0.17621, + "36": 0.17534, + "37": 0.17396, + "38": 0.17441, + "39": 0.17607, + "40": 0.17595, + "41": 0.17656, + "42": 0.17612, + "43": 0.17641, + "44": 0.17307, + "45": 0.17294, + "46": 0.17509, + "47": 0.23669, + "48": 0.17494, + "49": 0.1788, + "50": 0.1763, + "51": 0.31548, + "52": 0.94613, + "53": 0.1755, + "54": 0.17607, + "55": 0.17564, + "56": 0.177, + "57": 0.17532, + "58": 0.17696, + "59": 0.17608, + "60": 0.17259, + "61": 0.17576, + "62": 0.17488, + "63": 0.17372, + "64": 0.17599, + "65": 0.17292, + "66": 0.1765, + "67": 0.17439, + "68": 0.17684, + "69": 0.17641, + "70": 0.17839, + "71": 0.22013, + "72": 0.17798, + "73": 0.17733, + "74": 0.17683, + "75": 0.17752, + "76": 0.1786, + "77": 0.17652, + "78": 0.17675, + "79": 0.1765, + "80": 0.17914, + "81": 0.17389, + "82": 0.1766, + "83": 0.17661, + "84": 0.17501, + "85": 0.17681, + "86": 0.17677, + "87": 0.17914, + "88": 0.17952, + "89": 0.1787, + "90": 0.17876, + "91": 0.18084, + "92": 0.17689, + "93": 0.1755, + "94": 0.17653, + "95": 0.17924, + "96": 0.18006, + "97": 0.1771, + "98": 0.19423, + "99": 0.17532, + "100": 0.17549 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json index e706b8f5b63..bd119258613 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, - "3": 10.94862, + "1": 10.95177, + "2": 10.94867, + "3": 10.94863, "4": 10.94969, - "5": 10.94753, - "6": 10.94472, + "5": 10.94755, + "6": 10.94479, "7": 10.9465, - "8": 10.94472, - "9": 10.94186, - "10": 10.94475, + "8": 10.94479, + "9": 10.94191, + "10": 10.94479, "11": 10.94106, - "12": 10.93297, + "12": 10.93291, "13": 10.92308, - "14": 10.92593, - "15": 10.89969, - "16": 10.89084, - "17": 10.89814, - "18": 10.88337, - "19": 10.88981, - "20": 10.79731, - "21": 10.79299, - "22": 10.77996, - "23": 10.76964, - "24": 10.74573, - "25": 10.73536, - "26": 10.72512, - "27": 10.67795, - "28": 10.62002, - "29": 10.59158, - "30": 10.56328, - "31": 10.56514, - "32": 10.54392, - "33": 10.50914, - "34": 10.48321, - "35": 10.46994, - "36": 10.44683, - "37": 10.42648, - "38": 10.43048, - "39": 10.39271, - "40": 10.38041, - "41": 10.35136, - "42": 10.3275, + "14": 10.92589, + "15": 10.89963, + "16": 10.89081, + "17": 10.89822, + "18": 10.88344, + "19": 10.88987, + "20": 10.7974, + "21": 10.79296, + "22": 10.78008, + "23": 10.76968, + "24": 10.74582, + "25": 10.73534, + "26": 10.72506, + "27": 10.67793, + "28": 10.62001, + "29": 10.59163, + "30": 10.56335, + "31": 10.56519, + "32": 10.544, + "33": 10.50913, + "34": 10.48323, + "35": 10.4699, + "36": 10.44694, + "37": 10.42657, + "38": 10.43051, + "39": 10.39275, + "40": 10.3804, + "41": 10.35141, + "42": 10.32755, "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24783, - "47": 10.2438, + "44": 10.28348, + "45": 10.29608, + "46": 10.24785, + "47": 10.24385, "48": 10.19393, - "49": 10.19673, - "50": 10.18924, - "51": 10.19745, + "49": 10.19678, + "50": 10.18927, + "51": 10.19748, "52": 10.15223, - "53": 10.15768, - "54": 10.12071, - "55": 10.09673, - "56": 10.12108, - "57": 10.11169, - "58": 10.12097, - "59": 10.06243, - "60": 10.09231, - "61": 10.04173, - "62": 10.00957, - "63": 10.07951, - "64": 10.03237, - "65": 10.00321, - "66": 10.03953, - "67": 10.01958, - "68": 9.9857, - "69": 10.00195, + "53": 10.15775, + "54": 10.12079, + "55": 10.09676, + "56": 10.1211, + "57": 10.11179, + "58": 10.12101, + "59": 10.06249, + "60": 10.0924, + "61": 10.04177, + "62": 10.00959, + "63": 10.0795, + "64": 10.03239, + "65": 10.00322, + "66": 10.03956, + "67": 10.01964, + "68": 9.98577, + "69": 10.00204, "70": 9.98144, - "71": 10.00456, - "72": 9.99623, - "73": 9.9876, - "74": 9.97288, - "75": 9.93232, - "76": 9.96721, - "77": 9.96946, - "78": 9.92083, - "79": 9.91736, - "80": 9.93738, - "81": 9.95721, - "82": 9.89591, - "83": 9.86219, - "84": 9.80485, - "85": 9.78642, - "86": 9.89557, - "87": 9.90872, + "71": 10.00455, + "72": 9.99626, + "73": 9.98765, + "74": 9.97294, + "75": 9.93237, + "76": 9.9672, + "77": 9.96954, + "78": 9.92086, + "79": 9.9174, + "80": 9.93744, + "81": 9.95731, + "82": 9.89594, + "83": 9.86223, + "84": 9.80486, + "85": 9.78646, + "86": 9.89559, + "87": 9.90877, "88": 9.88858, - "89": 9.83328, - "90": 9.82422, - "91": 9.8393, - "92": 9.82245, - "93": 9.75866, - "94": 9.83495, - "95": 9.82146, - "96": 9.81157, - "97": 9.75625, - "98": 9.78244, - "99": 9.82335, - "100": 9.71583 + "89": 9.8333, + "90": 9.82427, + "91": 9.83932, + "92": 9.82252, + "93": 9.75876, + "94": 9.83501, + "95": 9.82152, + "96": 9.81164, + "97": 9.75629, + "98": 9.78246, + "99": 9.82339, + "100": 9.71587 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1733.0, - "2": 1654.0, - "3": 1807.0, - "4": 1633.0, - "5": 1831.0, - "6": 1678.0, - "7": 1895.0, - "8": 1671.0, - "9": 1673.0, - "10": 1708.0, - "11": 1553.0, - "12": 1673.0, - "13": 1817.0, - "14": 1834.0, - "15": 1646.0, - "16": 1768.0, - "17": 1815.0, - "18": 1755.0, - "19": 1673.0, - "20": 1652.0, - "21": 1774.0, - "22": 1666.0, - "23": 1742.0, - "24": 1813.0, - "25": 1613.0, - "26": 1757.0, - "27": 1933.0, - "28": 1878.0, - "29": 1897.0, - "30": 1773.0, - "31": 2012.0, - "32": 1945.0, - "33": 1954.0, - "34": 2032.0, - "35": 2079.0, - "36": 1910.0, - "37": 2215.0, - "38": 2091.0, - "39": 2118.0, - "40": 2207.0, - "41": 2215.0, - "42": 2146.0, - "43": 2268.0, - "44": 2277.0, - "45": 2471.0, - "46": 2361.0, - "47": 2478.0, - "48": 2516.0, - "49": 2742.0, - "50": 2642.0, - "51": 2514.0, - "52": 2737.0, - "53": 2633.0, - "54": 2812.0, - "55": 2551.0, - "56": 2785.0, - "57": 2305.0, - "58": 3578.0, - "59": 2929.0, - "60": 2969.0, - "61": 2767.0, - "62": 3207.0, - "63": 3261.0, - "64": 3564.0, - "65": 2658.0, - "66": 3028.0, - "67": 3825.0, - "68": 3302.0, - "69": 2965.0, - "70": 3273.0, - "71": 3178.0, - "72": 2916.0, - "73": 3501.0, - "74": 3278.0, - "75": 3294.0, - "76": 3359.0, - "77": 3682.0, - "78": 3230.0, - "79": 3223.0, - "80": 2932.0, - "81": 3215.0, - "82": 2894.0, - "83": 2944.0, - "84": 3031.0, - "85": 2667.0, - "86": 3211.0, - "87": 2910.0, - "88": 3105.0, - "89": 2985.0, - "90": 3930.0, - "91": 2757.0, - "92": 3004.0, - "93": 2976.0, - "94": 3091.0, - "95": 3323.0, - "96": 3464.0, - "97": 3570.0, - "98": 3215.0, - "99": 3459.0, - "100": 3386.0 + "1": 1720.0, + "2": 1610.0, + "3": 1759.0, + "4": 1672.0, + "5": 1790.0, + "6": 1645.0, + "7": 1951.0, + "8": 1673.0, + "9": 1638.0, + "10": 1711.0, + "11": 1665.0, + "12": 1693.0, + "13": 1756.0, + "14": 1856.0, + "15": 1588.0, + "16": 1726.0, + "17": 1789.0, + "18": 1794.0, + "19": 1682.0, + "20": 1650.0, + "21": 1797.0, + "22": 1651.0, + "23": 1734.0, + "24": 1800.0, + "25": 1706.0, + "26": 1766.0, + "27": 1833.0, + "28": 1796.0, + "29": 1794.0, + "30": 1818.0, + "31": 1985.0, + "32": 1935.0, + "33": 1949.0, + "34": 2021.0, + "35": 2052.0, + "36": 2020.0, + "37": 2106.0, + "38": 2035.0, + "39": 2189.0, + "40": 2178.0, + "41": 2319.0, + "42": 2030.0, + "43": 2423.0, + "44": 2211.0, + "45": 2387.0, + "46": 2376.0, + "47": 2480.0, + "48": 2587.0, + "49": 2711.0, + "50": 2618.0, + "51": 2475.0, + "52": 2747.0, + "53": 2656.0, + "54": 2815.0, + "55": 2536.0, + "56": 2716.0, + "57": 2263.0, + "58": 3635.0, + "59": 3047.0, + "60": 3031.0, + "61": 2806.0, + "62": 3246.0, + "63": 3289.0, + "64": 3593.0, + "65": 2540.0, + "66": 2944.0, + "67": 3836.0, + "68": 3485.0, + "69": 2941.0, + "70": 3355.0, + "71": 3196.0, + "72": 2909.0, + "73": 3498.0, + "74": 3224.0, + "75": 3120.0, + "76": 3269.0, + "77": 3648.0, + "78": 3293.0, + "79": 3169.0, + "80": 3022.0, + "81": 3305.0, + "82": 2934.0, + "83": 3071.0, + "84": 3060.0, + "85": 2653.0, + "86": 3079.0, + "87": 2941.0, + "88": 3158.0, + "89": 3045.0, + "90": 3795.0, + "91": 2844.0, + "92": 2887.0, + "93": 2965.0, + "94": 2974.0, + "95": 3328.0, + "96": 3516.0, + "97": 3595.0, + "98": 3304.0, + "99": 3431.0, + "100": 3362.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.80466, - "3": 0.18387, - "4": 0.18412, - "5": 0.18453, - "6": 0.18603, - "7": 0.1839, - "8": 0.18381, - "9": 0.18608, - "10": 0.18453, - "11": 0.18388, - "12": 0.18211, - "13": 0.18714, - "14": 0.18529, - "15": 0.1852, - "16": 0.1845, - "17": 0.18358, - "18": 0.18631, - "19": 0.18375, - "20": 0.18266, - "21": 0.18293, - "22": 0.18089, - "23": 0.18406, - "24": 0.18205, - "25": 0.18155, - "26": 0.18385, - "27": 0.18228, - "28": 0.18249, - "29": 0.1837, - "30": 0.18115, - "31": 0.17933, - "32": 0.18245, - "33": 0.18048, - "34": 0.17919, - "35": 0.18161, - "36": 0.18044, - "37": 0.18038, - "38": 0.18397, - "39": 0.18404, - "40": 0.1804, - "41": 0.18276, - "42": 0.18045, - "43": 0.18052, - "44": 0.18131, - "45": 0.18034, - "46": 0.18231, - "47": 0.18155, - "48": 0.17974, - "49": 0.18214, - "50": 0.1806, - "51": 0.20772, - "52": 0.79964, - "53": 0.18042, - "54": 0.18164, - "55": 0.18053, - "56": 0.18219, - "57": 0.17923, - "58": 0.18048, - "59": 0.18007, - "60": 0.18026, - "61": 0.17951, - "62": 0.17958, - "63": 0.18152, - "64": 0.17994, - "65": 0.17811, - "66": 0.17964, - "67": 0.17856, - "68": 0.18123, - "69": 0.18032, - "70": 0.17993, - "71": 0.17927, - "72": 0.17812, - "73": 0.18081, - "74": 0.18039, - "75": 0.18345, - "76": 0.17951, - "77": 0.17747, - "78": 0.17942, - "79": 0.17913, - "80": 0.18271, - "81": 0.17983, - "82": 0.17912, - "83": 0.18108, - "84": 0.18144, - "85": 0.18107, - "86": 0.18062, - "87": 0.17977, - "88": 0.18167, - "89": 0.18068, - "90": 0.18019, - "91": 0.18019, - "92": 0.17872, - "93": 0.17945, - "94": 0.17979, - "95": 0.17803, - "96": 0.18154, - "97": 0.17801, - "98": 0.18046, - "99": 0.18003, - "100": 0.22411 + "2": 5.40071, + "3": 0.19571, + "4": 0.19596, + "5": 0.19504, + "6": 0.19501, + "7": 0.19317, + "8": 0.19293, + "9": 0.19199, + "10": 0.19253, + "11": 0.19472, + "12": 0.19385, + "13": 0.19583, + "14": 0.18989, + "15": 0.1911, + "16": 0.18991, + "17": 0.19213, + "18": 0.18952, + "19": 0.18896, + "20": 0.19292, + "21": 0.19156, + "22": 0.19269, + "23": 0.19276, + "24": 0.19025, + "25": 0.1911, + "26": 0.19094, + "27": 0.1932, + "28": 0.19219, + "29": 0.19481, + "30": 0.19005, + "31": 0.19051, + "32": 0.19047, + "33": 0.19361, + "34": 0.19024, + "35": 0.19079, + "36": 0.192, + "37": 0.19178, + "38": 0.19243, + "39": 0.19229, + "40": 0.19366, + "41": 0.19259, + "42": 0.19127, + "43": 0.19187, + "44": 0.18976, + "45": 0.19, + "46": 0.19133, + "47": 0.19837, + "48": 0.19061, + "49": 0.19075, + "50": 0.19014, + "51": 0.21907, + "52": 0.81422, + "53": 0.19803, + "54": 0.19151, + "55": 0.19347, + "56": 0.19311, + "57": 0.19592, + "58": 0.20128, + "59": 0.19826, + "60": 0.19114, + "61": 0.1934, + "62": 0.19415, + "63": 0.19599, + "64": 0.19371, + "65": 0.19082, + "66": 0.19302, + "67": 0.19071, + "68": 0.19156, + "69": 0.19308, + "70": 0.19283, + "71": 0.1914, + "72": 0.19282, + "73": 0.19589, + "74": 0.19233, + "75": 0.19396, + "76": 0.19128, + "77": 0.19243, + "78": 0.19123, + "79": 0.19137, + "80": 0.19125, + "81": 0.19146, + "82": 0.19116, + "83": 0.19134, + "84": 0.19073, + "85": 0.1932, + "86": 0.19692, + "87": 0.19631, + "88": 0.19515, + "89": 0.19068, + "90": 0.19169, + "91": 0.19176, + "92": 0.19037, + "93": 0.19158, + "94": 0.19526, + "95": 0.19237, + "96": 0.19019, + "97": 0.19254, + "98": 0.19264, + "99": 0.19531, + "100": 0.1941 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json index 61b91e9f7fa..51609a66f3a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92866, - "4": 10.9288, - "5": 10.93088, - "6": 10.92771, - "7": 10.92336, - "8": 10.92159, - "9": 10.92031, + "2": 10.92832, + "3": 10.92864, + "4": 10.92888, + "5": 10.93089, + "6": 10.92772, + "7": 10.92341, + "8": 10.92161, + "9": 10.92027, "10": 10.91964, - "11": 10.91027, - "12": 10.91814, - "13": 10.91523, - "14": 10.89825, - "15": 10.88042, - "16": 10.86841, + "11": 10.91024, + "12": 10.91826, + "13": 10.91526, + "14": 10.89832, + "15": 10.88051, + "16": 10.8684, "17": 10.86882, - "18": 10.86258, - "19": 10.86185, - "20": 10.77348, - "21": 10.75516, - "22": 10.75521, - "23": 10.74508, - "24": 10.71106, - "25": 10.71082, - "26": 10.69968, - "27": 10.65192, - "28": 10.58249, - "29": 10.55636, - "30": 10.54863, - "31": 10.52499, - "32": 10.51761, - "33": 10.48144, - "34": 10.451, - "35": 10.44994, - "36": 10.43486, - "37": 10.39411, - "38": 10.39546, - "39": 10.36016, - "40": 10.35798, - "41": 10.32782, - "42": 10.30704, - "43": 10.28206, - "44": 10.25604, - "45": 10.27039, - "46": 10.23404, - "47": 10.21882, + "18": 10.86249, + "19": 10.86187, + "20": 10.77339, + "21": 10.75518, + "22": 10.75531, + "23": 10.74509, + "24": 10.71112, + "25": 10.71086, + "26": 10.69962, + "27": 10.65199, + "28": 10.58247, + "29": 10.55645, + "30": 10.54865, + "31": 10.52506, + "32": 10.51769, + "33": 10.48149, + "34": 10.45111, + "35": 10.44997, + "36": 10.43493, + "37": 10.3941, + "38": 10.39549, + "39": 10.36015, + "40": 10.35806, + "41": 10.32785, + "42": 10.30705, + "43": 10.28217, + "44": 10.25607, + "45": 10.27045, + "46": 10.23413, + "47": 10.21893, "48": 10.17023, - "49": 10.1759, - "50": 10.17855, - "51": 10.19228, - "52": 10.13723, - "53": 10.13552, - "54": 10.10521, - "55": 10.08055, - "56": 10.11116, - "57": 10.09599, - "58": 10.10931, - "59": 10.05623, - "60": 10.07399, - "61": 10.02775, - "62": 10.0003, - "63": 10.07322, - "64": 10.03695, - "65": 10.00341, - "66": 10.03038, - "67": 10.0072, - "68": 9.97019, - "69": 9.9915, - "70": 9.97453, - "71": 10.00065, - "72": 9.97779, - "73": 9.97166, - "74": 9.95604, - "75": 9.93149, - "76": 9.96344, - "77": 9.95639, - "78": 9.90552, - "79": 9.91218, - "80": 9.92898, - "81": 9.95655, - "82": 9.89202, - "83": 9.85589, - "84": 9.78695, - "85": 9.78572, - "86": 9.88392, - "87": 9.91028, - "88": 9.88719, - "89": 9.81819, - "90": 9.81249, - "91": 9.82779, - "92": 9.81659, - "93": 9.74998, - "94": 9.8309, - "95": 9.82648, - "96": 9.80702, - "97": 9.74393, - "98": 9.77873, - "99": 9.82058, - "100": 9.71398 + "49": 10.17594, + "50": 10.17859, + "51": 10.19235, + "52": 10.13725, + "53": 10.13558, + "54": 10.10524, + "55": 10.08058, + "56": 10.11123, + "57": 10.09601, + "58": 10.10934, + "59": 10.05632, + "60": 10.07401, + "61": 10.02777, + "62": 10.00036, + "63": 10.07317, + "64": 10.03696, + "65": 10.00345, + "66": 10.0304, + "67": 10.00724, + "68": 9.97024, + "69": 9.99153, + "70": 9.9746, + "71": 10.00066, + "72": 9.97785, + "73": 9.9717, + "74": 9.95607, + "75": 9.93157, + "76": 9.96345, + "77": 9.95644, + "78": 9.90557, + "79": 9.9122, + "80": 9.92906, + "81": 9.95666, + "82": 9.89201, + "83": 9.85599, + "84": 9.78701, + "85": 9.7858, + "86": 9.88398, + "87": 9.91034, + "88": 9.88728, + "89": 9.81818, + "90": 9.81251, + "91": 9.82784, + "92": 9.81667, + "93": 9.75004, + "94": 9.83094, + "95": 9.82651, + "96": 9.80705, + "97": 9.74396, + "98": 9.77876, + "99": 9.82056, + "100": 9.71405 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1731.0, - "2": 1691.0, - "3": 1669.0, - "4": 1581.0, - "5": 1644.0, - "6": 1685.0, - "7": 1815.0, - "8": 1709.0, - "9": 1781.0, - "10": 1748.0, - "11": 1630.0, - "12": 1732.0, - "13": 1742.0, - "14": 1815.0, - "15": 1549.0, - "16": 1654.0, - "17": 1749.0, - "18": 1742.0, - "19": 1686.0, - "20": 1643.0, - "21": 1695.0, - "22": 1733.0, - "23": 1658.0, - "24": 1781.0, - "25": 1666.0, - "26": 1789.0, - "27": 1777.0, - "28": 1803.0, - "29": 1831.0, - "30": 1831.0, - "31": 2000.0, - "32": 1953.0, - "33": 1927.0, - "34": 2012.0, - "35": 2071.0, - "36": 1876.0, - "37": 2178.0, - "38": 2133.0, - "39": 2155.0, - "40": 2357.0, - "41": 2339.0, - "42": 1887.0, - "43": 2351.0, - "44": 2221.0, - "45": 2475.0, - "46": 2480.0, - "47": 2497.0, + "1": 1684.0, + "2": 1628.0, + "3": 1753.0, + "4": 1661.0, + "5": 1709.0, + "6": 1619.0, + "7": 1824.0, + "8": 1706.0, + "9": 1707.0, + "10": 1648.0, + "11": 1619.0, + "12": 1773.0, + "13": 1782.0, + "14": 1928.0, + "15": 1597.0, + "16": 1623.0, + "17": 1698.0, + "18": 1750.0, + "19": 1626.0, + "20": 1618.0, + "21": 1771.0, + "22": 1706.0, + "23": 1626.0, + "24": 1718.0, + "25": 1588.0, + "26": 1757.0, + "27": 1832.0, + "28": 1744.0, + "29": 1941.0, + "30": 1853.0, + "31": 1889.0, + "32": 1871.0, + "33": 1977.0, + "34": 2042.0, + "35": 2009.0, + "36": 1943.0, + "37": 2226.0, + "38": 2146.0, + "39": 2107.0, + "40": 2290.0, + "41": 2277.0, + "42": 1917.0, + "43": 2428.0, + "44": 2234.0, + "45": 2588.0, + "46": 2488.0, + "47": 2491.0, "48": 2601.0, - "49": 2909.0, - "50": 2638.0, - "51": 2495.0, - "52": 2790.0, - "53": 2729.0, - "54": 2777.0, - "55": 2626.0, - "56": 2743.0, - "57": 2177.0, - "58": 3682.0, - "59": 2899.0, - "60": 2899.0, - "61": 2826.0, - "62": 3369.0, - "63": 3377.0, - "64": 3714.0, - "65": 2750.0, - "66": 3201.0, - "67": 3778.0, - "68": 3542.0, - "69": 3015.0, - "70": 3264.0, - "71": 3129.0, - "72": 3054.0, - "73": 3365.0, - "74": 3234.0, - "75": 3345.0, - "76": 3367.0, - "77": 3958.0, - "78": 3329.0, - "79": 3194.0, - "80": 2951.0, - "81": 3592.0, - "82": 2943.0, - "83": 3073.0, - "84": 3037.0, - "85": 2541.0, - "86": 3023.0, - "87": 2916.0, - "88": 3027.0, - "89": 3180.0, - "90": 3778.0, - "91": 3014.0, - "92": 2726.0, - "93": 3051.0, - "94": 3000.0, - "95": 3189.0, - "96": 3382.0, - "97": 3490.0, - "98": 3305.0, - "99": 3298.0, - "100": 3311.0 + "49": 2860.0, + "50": 2622.0, + "51": 2566.0, + "52": 2713.0, + "53": 2694.0, + "54": 2837.0, + "55": 2660.0, + "56": 2691.0, + "57": 2100.0, + "58": 3764.0, + "59": 2897.0, + "60": 3048.0, + "61": 2813.0, + "62": 3200.0, + "63": 3426.0, + "64": 3603.0, + "65": 2607.0, + "66": 3090.0, + "67": 3762.0, + "68": 3476.0, + "69": 3189.0, + "70": 3310.0, + "71": 3107.0, + "72": 3040.0, + "73": 3459.0, + "74": 3262.0, + "75": 3167.0, + "76": 3312.0, + "77": 3909.0, + "78": 3290.0, + "79": 3200.0, + "80": 3133.0, + "81": 3597.0, + "82": 2947.0, + "83": 3172.0, + "84": 2917.0, + "85": 2518.0, + "86": 3140.0, + "87": 2832.0, + "88": 3076.0, + "89": 3067.0, + "90": 3800.0, + "91": 3056.0, + "92": 2804.0, + "93": 3105.0, + "94": 3146.0, + "95": 3205.0, + "96": 3332.0, + "97": 3391.0, + "98": 3361.0, + "99": 3318.0, + "100": 3245.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.06977, - "3": 2.18555, - "4": 2.0122, - "5": 1.5422, - "6": 1.88019, - "7": 2.20797, - "8": 2.71542, - "9": 2.10583, - "10": 2.53978, - "11": 2.52922, - "12": 1.72555, - "13": 1.99992, - "14": 2.65834, - "15": 1.93394, - "16": 2.06383, - "17": 2.36017, - "18": 2.0535, - "19": 1.90883, - "20": 2.08271, - "21": 2.25995, - "22": 1.92834, - "23": 1.74054, - "24": 1.70254, - "25": 2.54029, - "26": 2.39901, - "27": 1.99731, - "28": 1.98739, - "29": 1.50029, - "30": 1.96645, - "31": 2.15154, - "32": 2.56678, - "33": 1.99054, - "34": 2.13667, - "35": 2.64511, - "36": 2.09883, - "37": 1.91001, - "38": 1.5396, - "39": 2.46548, - "40": 1.9233, - "41": 2.28732, - "42": 2.35301, - "43": 1.65789, - "44": 1.56099, - "45": 2.05719, - "46": 1.70137, - "47": 1.68981, - "48": 1.92915, - "49": 1.66933, - "50": 1.88296, - "51": 1.23599, - "52": 2.07863, - "53": 1.86058, - "54": 1.69935, - "55": 1.78663, - "56": 2.13775, - "57": 1.19895, - "58": 2.12168, - "59": 1.45477, - "60": 1.47497, - "61": 1.904, - "62": 2.27901, - "63": 1.59531, - "64": 2.3399, - "65": 2.91356, - "66": 2.56806, - "67": 2.2204, - "68": 2.10613, - "69": 1.99613, - "70": 2.2961, - "71": 2.4294, - "72": 2.12526, - "73": 2.02606, - "74": 1.36499, - "75": 2.07417, - "76": 1.94247, - "77": 1.9907, - "78": 1.86404, - "79": 2.14268, - "80": 1.72401, - "81": 2.07887, - "82": 1.62928, - "83": 1.88239, - "84": 1.52229, - "85": 1.61961, - "86": 1.78662, - "87": 2.11775, - "88": 1.8844, - "89": 1.71247, - "90": 1.66331, - "91": 1.76112, - "92": 1.83053, - "93": 1.41142, - "94": 1.51544, - "95": 1.81539, - "96": 1.68572, - "97": 1.69888, - "98": 1.88079, - "99": 2.63883, - "100": 1.59243 + "2": 5.07125, + "3": 2.31098, + "4": 2.82088, + "5": 2.02478, + "6": 2.56278, + "7": 2.04359, + "8": 2.64414, + "9": 3.00566, + "10": 2.44323, + "11": 2.64929, + "12": 2.00885, + "13": 1.9096, + "14": 2.36206, + "15": 1.97762, + "16": 2.14053, + "17": 2.64211, + "18": 1.80678, + "19": 1.56087, + "20": 1.58349, + "21": 2.15188, + "22": 1.72435, + "23": 1.77659, + "24": 1.7121, + "25": 2.13609, + "26": 2.27878, + "27": 1.65034, + "28": 1.60789, + "29": 1.45543, + "30": 1.766, + "31": 1.35499, + "32": 2.4081, + "33": 1.3692, + "34": 1.50576, + "35": 2.05773, + "36": 1.60003, + "37": 1.43307, + "38": 1.52596, + "39": 1.8616, + "40": 1.53284, + "41": 1.87365, + "42": 1.62293, + "43": 1.94555, + "44": 1.95807, + "45": 2.69226, + "46": 2.29999, + "47": 2.37463, + "48": 2.17134, + "49": 1.6068, + "50": 2.48683, + "51": 1.43599, + "52": 1.70124, + "53": 2.42209, + "54": 2.34181, + "55": 1.80522, + "56": 2.77867, + "57": 1.53732, + "58": 2.38305, + "59": 2.21083, + "60": 1.86941, + "61": 2.43504, + "62": 2.31375, + "63": 1.68253, + "64": 2.26141, + "65": 1.79426, + "66": 1.93816, + "67": 2.1352, + "68": 1.55142, + "69": 1.52064, + "70": 1.64749, + "71": 2.53934, + "72": 1.73134, + "73": 1.71005, + "74": 1.78592, + "75": 1.56218, + "76": 1.58404, + "77": 1.36456, + "78": 1.16372, + "79": 1.81909, + "80": 1.61289, + "81": 2.04292, + "82": 2.25666, + "83": 1.74502, + "84": 1.50738, + "85": 1.68232, + "86": 1.37655, + "87": 2.16292, + "88": 1.8937, + "89": 1.83243, + "90": 1.78226, + "91": 2.16331, + "92": 1.76002, + "93": 2.00611, + "94": 1.77846, + "95": 2.55852, + "96": 2.20079, + "97": 1.44984, + "98": 1.69142, + "99": 2.26456, + "100": 2.14084 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json index c71ef2a52d3..6c80dc49089 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92942, - "2": 10.92777, - "3": 10.92678, - "4": 10.92616, - "5": 10.9267, - "6": 10.92764, - "7": 10.92398, - "8": 10.91832, - "9": 10.91964, - "10": 10.91772, - "11": 10.9081, - "12": 10.91698, - "13": 10.91063, - "14": 10.90207, - "15": 10.88113, - "16": 10.87325, - "17": 10.86927, + "1": 10.92954, + "2": 10.92779, + "3": 10.92675, + "4": 10.92617, + "5": 10.92685, + "6": 10.92766, + "7": 10.92394, + "8": 10.91823, + "9": 10.91968, + "10": 10.91783, + "11": 10.90805, + "12": 10.91691, + "13": 10.91061, + "14": 10.90208, + "15": 10.88117, + "16": 10.87323, + "17": 10.86924, "18": 10.86347, "19": 10.86027, - "20": 10.7704, - "21": 10.7558, - "22": 10.75377, - "23": 10.74581, + "20": 10.77048, + "21": 10.75588, + "22": 10.75378, + "23": 10.74582, "24": 10.71439, - "25": 10.71349, - "26": 10.7028, - "27": 10.65473, - "28": 10.57862, - "29": 10.55897, - "30": 10.54749, - "31": 10.52972, - "32": 10.51715, - "33": 10.48067, - "34": 10.45115, - "35": 10.45041, - "36": 10.43318, - "37": 10.39352, + "25": 10.71356, + "26": 10.70281, + "27": 10.65478, + "28": 10.57866, + "29": 10.55902, + "30": 10.54751, + "31": 10.52974, + "32": 10.51726, + "33": 10.48074, + "34": 10.45122, + "35": 10.45042, + "36": 10.43325, + "37": 10.39358, "38": 10.39541, - "39": 10.36368, - "40": 10.35672, - "41": 10.32573, - "42": 10.3077, - "43": 10.28179, - "44": 10.25427, - "45": 10.2702, - "46": 10.23518, - "47": 10.21978, - "48": 10.16978, - "49": 10.17822, - "50": 10.17837, - "51": 10.1923, - "52": 10.13727, - "53": 10.13533, - "54": 10.10653, - "55": 10.08096, - "56": 10.11297, - "57": 10.09527, - "58": 10.10881, - "59": 10.05601, - "60": 10.07522, - "61": 10.02733, - "62": 10.00227, - "63": 10.07213, - "64": 10.03491, - "65": 10.00278, - "66": 10.02915, - "67": 10.00666, + "39": 10.36364, + "40": 10.35673, + "41": 10.32578, + "42": 10.30776, + "43": 10.28189, + "44": 10.25432, + "45": 10.27025, + "46": 10.23525, + "47": 10.21977, + "48": 10.1698, + "49": 10.17827, + "50": 10.17844, + "51": 10.19231, + "52": 10.13728, + "53": 10.1354, + "54": 10.10661, + "55": 10.08098, + "56": 10.11295, + "57": 10.09534, + "58": 10.10889, + "59": 10.05609, + "60": 10.07523, + "61": 10.02741, + "62": 10.00226, + "63": 10.07217, + "64": 10.03496, + "65": 10.00276, + "66": 10.02921, + "67": 10.00669, "68": 9.96974, - "69": 9.99041, - "70": 9.97339, - "71": 10.00125, - "72": 9.97761, + "69": 9.99044, + "70": 9.97337, + "71": 10.00127, + "72": 9.97765, "73": 9.97164, - "74": 9.95659, - "75": 9.92901, - "76": 9.96291, - "77": 9.9572, - "78": 9.90596, - "79": 9.90998, - "80": 9.93062, - "81": 9.95611, - "82": 9.89121, - "83": 9.85516, - "84": 9.78771, - "85": 9.78742, - "86": 9.88333, - "87": 9.91031, + "74": 9.95662, + "75": 9.92907, + "76": 9.96297, + "77": 9.95726, + "78": 9.90604, + "79": 9.91006, + "80": 9.93069, + "81": 9.95613, + "82": 9.89127, + "83": 9.85522, + "84": 9.78775, + "85": 9.78743, + "86": 9.88339, + "87": 9.91033, "88": 9.88594, - "89": 9.81829, - "90": 9.8142, - "91": 9.82558, - "92": 9.81425, - "93": 9.75216, + "89": 9.81827, + "90": 9.81423, + "91": 9.82562, + "92": 9.81426, + "93": 9.75215, "94": 9.83216, - "95": 9.82474, - "96": 9.80679, + "95": 9.82478, + "96": 9.80682, "97": 9.74328, - "98": 9.77878, - "99": 9.81992, - "100": 9.71216 + "98": 9.77881, + "99": 9.81997, + "100": 9.71224 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1838.0, - "2": 1679.0, - "3": 1823.0, - "4": 1657.0, - "5": 1659.0, - "6": 1659.0, - "7": 1901.0, - "8": 1778.0, - "9": 1713.0, - "10": 1665.0, - "11": 1674.0, - "12": 1579.0, - "13": 1665.0, - "14": 1874.0, - "15": 1587.0, - "16": 1603.0, - "17": 1719.0, - "18": 1724.0, - "19": 1650.0, - "20": 1639.0, - "21": 1743.0, - "22": 1788.0, - "23": 1639.0, - "24": 1761.0, - "25": 1703.0, - "26": 1754.0, - "27": 1780.0, - "28": 1879.0, - "29": 1887.0, - "30": 1815.0, - "31": 1949.0, - "32": 1992.0, - "33": 1933.0, - "34": 1838.0, - "35": 1997.0, - "36": 1945.0, - "37": 2203.0, - "38": 2064.0, - "39": 2141.0, - "40": 2254.0, - "41": 2283.0, - "42": 2037.0, - "43": 2343.0, - "44": 2171.0, - "45": 2487.0, - "46": 2526.0, - "47": 2456.0, - "48": 2611.0, - "49": 2921.0, - "50": 2600.0, - "51": 2510.0, - "52": 2852.0, - "53": 2598.0, - "54": 2774.0, - "55": 2609.0, - "56": 2730.0, - "57": 2196.0, - "58": 3537.0, - "59": 2868.0, - "60": 3030.0, - "61": 2835.0, - "62": 3206.0, - "63": 3374.0, - "64": 3656.0, - "65": 2646.0, - "66": 2944.0, - "67": 3862.0, - "68": 3436.0, - "69": 3030.0, - "70": 3271.0, - "71": 3210.0, - "72": 3057.0, - "73": 3453.0, - "74": 3277.0, - "75": 3257.0, - "76": 3250.0, - "77": 3639.0, - "78": 3207.0, - "79": 3079.0, - "80": 3041.0, - "81": 3578.0, - "82": 2879.0, - "83": 3166.0, - "84": 2924.0, - "85": 2537.0, - "86": 3013.0, - "87": 2973.0, - "88": 3029.0, - "89": 2997.0, - "90": 3837.0, - "91": 3034.0, - "92": 2813.0, - "93": 3084.0, - "94": 2992.0, - "95": 3249.0, - "96": 3345.0, - "97": 3465.0, - "98": 3255.0, - "99": 3305.0, - "100": 3279.0 + "1": 1798.0, + "2": 1730.0, + "3": 1779.0, + "4": 1679.0, + "5": 1748.0, + "6": 1640.0, + "7": 1794.0, + "8": 1728.0, + "9": 1693.0, + "10": 1809.0, + "11": 1614.0, + "12": 1639.0, + "13": 1847.0, + "14": 1794.0, + "15": 1715.0, + "16": 1667.0, + "17": 1613.0, + "18": 1709.0, + "19": 1627.0, + "20": 1653.0, + "21": 1719.0, + "22": 1663.0, + "23": 1603.0, + "24": 1772.0, + "25": 1718.0, + "26": 1739.0, + "27": 1804.0, + "28": 1814.0, + "29": 1882.0, + "30": 1745.0, + "31": 1942.0, + "32": 1894.0, + "33": 1996.0, + "34": 1897.0, + "35": 2072.0, + "36": 1985.0, + "37": 2210.0, + "38": 2076.0, + "39": 2251.0, + "40": 2205.0, + "41": 2262.0, + "42": 1978.0, + "43": 2386.0, + "44": 2219.0, + "45": 2468.0, + "46": 2374.0, + "47": 2386.0, + "48": 2744.0, + "49": 2913.0, + "50": 2603.0, + "51": 2527.0, + "52": 2843.0, + "53": 2670.0, + "54": 2854.0, + "55": 2552.0, + "56": 2726.0, + "57": 2248.0, + "58": 3587.0, + "59": 2898.0, + "60": 3025.0, + "61": 2849.0, + "62": 3243.0, + "63": 3421.0, + "64": 3474.0, + "65": 2682.0, + "66": 2921.0, + "67": 3813.0, + "68": 3404.0, + "69": 3094.0, + "70": 3315.0, + "71": 3073.0, + "72": 2999.0, + "73": 3614.0, + "74": 3364.0, + "75": 3332.0, + "76": 3326.0, + "77": 3721.0, + "78": 3282.0, + "79": 3191.0, + "80": 2947.0, + "81": 3677.0, + "82": 2893.0, + "83": 3141.0, + "84": 2992.0, + "85": 2540.0, + "86": 3014.0, + "87": 2909.0, + "88": 2909.0, + "89": 3095.0, + "90": 3741.0, + "91": 2950.0, + "92": 2856.0, + "93": 3015.0, + "94": 3098.0, + "95": 3318.0, + "96": 3395.0, + "97": 3654.0, + "98": 3093.0, + "99": 3350.0, + "100": 3189.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 514359808.0, - "2": 514359808.0, - "3": 514359808.0, - "4": 514359808.0, - "5": 514359808.0, - "6": 514359808.0, - "7": 514359808.0, - "8": 514359808.0, - "9": 514359808.0, - "10": 514359808.0, - "11": 514359808.0, - "12": 514359808.0, - "13": 514359808.0, - "14": 514359808.0, - "15": 514359808.0, - "16": 514359808.0, - "17": 514359808.0, - "18": 514359808.0, - "19": 514359808.0, - "20": 514359808.0, - "21": 514359808.0, - "22": 514359808.0, - "23": 514359808.0, - "24": 514359808.0, - "25": 514359808.0, - "26": 514359808.0, - "27": 514359808.0, - "28": 514359808.0, - "29": 514359808.0, - "30": 514359808.0, - "31": 514359808.0, - "32": 514359808.0, - "33": 514359808.0, - "34": 514359808.0, - "35": 514359808.0, - "36": 514359808.0, - "37": 514359808.0, - "38": 514359808.0, - "39": 514359808.0, - "40": 514359808.0, - "41": 514359808.0, - "42": 514359808.0, - "43": 514359808.0, - "44": 514359808.0, - "45": 514359808.0, - "46": 514359808.0, - "47": 514359808.0, - "48": 514359808.0, - "49": 514359808.0, - "50": 514359808.0, - "51": 514359808.0, - "52": 514359808.0, - "53": 514359808.0, - "54": 514359808.0, - "55": 514359808.0, - "56": 514359808.0, - "57": 514359808.0, - "58": 514359808.0, - "59": 514359808.0, - "60": 514359808.0, - "61": 514359808.0, - "62": 514359808.0, - "63": 514359808.0, - "64": 514359808.0, - "65": 514359808.0, - "66": 514359808.0, - "67": 514359808.0, - "68": 514359808.0, - "69": 514359808.0, - "70": 514359808.0, - "71": 514359808.0, - "72": 514359808.0, - "73": 514359808.0, - "74": 514359808.0, - "75": 514359808.0, - "76": 514359808.0, - "77": 514359808.0, - "78": 514359808.0, - "79": 514359808.0, - "80": 514359808.0, - "81": 514359808.0, - "82": 514359808.0, - "83": 514359808.0, - "84": 514359808.0, - "85": 514359808.0, - "86": 514359808.0, - "87": 514359808.0, - "88": 514359808.0, - "89": 514359808.0, - "90": 514359808.0, - "91": 514359808.0, - "92": 514359808.0, - "93": 514359808.0, - "94": 514359808.0, - "95": 514359808.0, - "96": 514359808.0, - "97": 514359808.0, - "98": 514359808.0, - "99": 514359808.0, - "100": 514359808.0 + "1": 514884096.0, + "2": 514884096.0, + "3": 514884096.0, + "4": 514884096.0, + "5": 514884096.0, + "6": 514884096.0, + "7": 514884096.0, + "8": 514884096.0, + "9": 514884096.0, + "10": 514884096.0, + "11": 514884096.0, + "12": 514884096.0, + "13": 514884096.0, + "14": 514884096.0, + "15": 514884096.0, + "16": 514884096.0, + "17": 514884096.0, + "18": 514884096.0, + "19": 514884096.0, + "20": 514884096.0, + "21": 514884096.0, + "22": 514884096.0, + "23": 514884096.0, + "24": 514884096.0, + "25": 514884096.0, + "26": 514884096.0, + "27": 514884096.0, + "28": 514884096.0, + "29": 514884096.0, + "30": 514884096.0, + "31": 514884096.0, + "32": 514884096.0, + "33": 514884096.0, + "34": 514884096.0, + "35": 514884096.0, + "36": 514884096.0, + "37": 514884096.0, + "38": 514884096.0, + "39": 514884096.0, + "40": 514884096.0, + "41": 514884096.0, + "42": 514884096.0, + "43": 514884096.0, + "44": 514884096.0, + "45": 514884096.0, + "46": 514884096.0, + "47": 514884096.0, + "48": 514884096.0, + "49": 514884096.0, + "50": 514884096.0, + "51": 514884096.0, + "52": 514884096.0, + "53": 514884096.0, + "54": 514884096.0, + "55": 514884096.0, + "56": 514884096.0, + "57": 514884096.0, + "58": 514884096.0, + "59": 514884096.0, + "60": 514884096.0, + "61": 514884096.0, + "62": 514884096.0, + "63": 514884096.0, + "64": 514884096.0, + "65": 514884096.0, + "66": 514884096.0, + "67": 514884096.0, + "68": 514884096.0, + "69": 514884096.0, + "70": 514884096.0, + "71": 514884096.0, + "72": 514884096.0, + "73": 514884096.0, + "74": 514884096.0, + "75": 514884096.0, + "76": 514884096.0, + "77": 514884096.0, + "78": 514884096.0, + "79": 514884096.0, + "80": 514884096.0, + "81": 514884096.0, + "82": 514884096.0, + "83": 514884096.0, + "84": 514884096.0, + "85": 514884096.0, + "86": 514884096.0, + "87": 514884096.0, + "88": 514884096.0, + "89": 514884096.0, + "90": 514884096.0, + "91": 514884096.0, + "92": 514884096.0, + "93": 514884096.0, + "94": 514884096.0, + "95": 514884096.0, + "96": 514884096.0, + "97": 514884096.0, + "98": 514884096.0, + "99": 514884096.0, + "100": 514884096.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1258060288.0, - "2": 1437084160.0, - "3": 1437084160.0, - "4": 1437084160.0, - "5": 1437084160.0, - "6": 1437084160.0, - "7": 1437084160.0, - "8": 1437084160.0, - "9": 1437084160.0, - "10": 1437084160.0, - "11": 1437084160.0, - "12": 1437084160.0, - "13": 1437084160.0, - "14": 1437084160.0, - "15": 1437084160.0, - "16": 1437084160.0, - "17": 1437084160.0, - "18": 1437084160.0, - "19": 1437084160.0, - "20": 1437084160.0, - "21": 1437084160.0, - "22": 1437084160.0, - "23": 1437084160.0, - "24": 1437084160.0, - "25": 1437084160.0, - "26": 1437084160.0, - "27": 1437084160.0, - "28": 1437084160.0, - "29": 1437084160.0, - "30": 1437084160.0, - "31": 1437084160.0, - "32": 1437084160.0, - "33": 1437084160.0, - "34": 1437084160.0, - "35": 1437084160.0, - "36": 1437084160.0, - "37": 1437084160.0, - "38": 1437084160.0, - "39": 1437084160.0, - "40": 1437084160.0, - "41": 1437084160.0, - "42": 1437084160.0, - "43": 1437084160.0, - "44": 1437084160.0, - "45": 1437084160.0, - "46": 1437084160.0, - "47": 1437084160.0, - "48": 1437084160.0, - "49": 1437084160.0, - "50": 1437084160.0, - "51": 1437084160.0, - "52": 1437084160.0, - "53": 1437084160.0, - "54": 1437084160.0, - "55": 1437084160.0, - "56": 1437084160.0, - "57": 1437084160.0, - "58": 1437084160.0, - "59": 1437084160.0, - "60": 1437084160.0, - "61": 1437084160.0, - "62": 1437084160.0, - "63": 1437084160.0, - "64": 1437084160.0, - "65": 1437084160.0, - "66": 1437084160.0, - "67": 1437084160.0, - "68": 1437084160.0, - "69": 1437084160.0, - "70": 1437084160.0, - "71": 1437084160.0, - "72": 1437084160.0, - "73": 1437084160.0, - "74": 1437084160.0, - "75": 1437084160.0, - "76": 1437084160.0, - "77": 1437084160.0, - "78": 1437084160.0, - "79": 1437084160.0, - "80": 1437084160.0, - "81": 1437084160.0, - "82": 1437084160.0, - "83": 1437084160.0, - "84": 1437084160.0, - "85": 1437084160.0, - "86": 1437084160.0, - "87": 1437084160.0, - "88": 1437084160.0, - "89": 1437084160.0, - "90": 1437084160.0, - "91": 1437084160.0, - "92": 1437084160.0, - "93": 1437084160.0, - "94": 1437084160.0, - "95": 1437084160.0, - "96": 1437084160.0, - "97": 1437084160.0, - "98": 1437084160.0, - "99": 1437084160.0, - "100": 1437084160.0 + "1": 1259108864.0, + "2": 1438394880.0, + "3": 1438394880.0, + "4": 1438394880.0, + "5": 1438394880.0, + "6": 1438394880.0, + "7": 1438394880.0, + "8": 1438394880.0, + "9": 1438394880.0, + "10": 1438394880.0, + "11": 1438394880.0, + "12": 1438394880.0, + "13": 1438394880.0, + "14": 1438394880.0, + "15": 1438394880.0, + "16": 1438394880.0, + "17": 1438394880.0, + "18": 1438394880.0, + "19": 1438394880.0, + "20": 1438394880.0, + "21": 1438394880.0, + "22": 1438394880.0, + "23": 1438394880.0, + "24": 1438394880.0, + "25": 1438394880.0, + "26": 1438394880.0, + "27": 1438394880.0, + "28": 1438394880.0, + "29": 1438394880.0, + "30": 1438394880.0, + "31": 1438394880.0, + "32": 1438394880.0, + "33": 1438394880.0, + "34": 1438394880.0, + "35": 1438394880.0, + "36": 1438394880.0, + "37": 1438394880.0, + "38": 1438394880.0, + "39": 1438394880.0, + "40": 1438394880.0, + "41": 1438394880.0, + "42": 1438394880.0, + "43": 1438394880.0, + "44": 1438394880.0, + "45": 1438394880.0, + "46": 1438394880.0, + "47": 1438394880.0, + "48": 1438394880.0, + "49": 1438394880.0, + "50": 1438394880.0, + "51": 1439181312.0, + "52": 1439181312.0, + "53": 1439181312.0, + "54": 1439181312.0, + "55": 1439181312.0, + "56": 1439181312.0, + "57": 1439181312.0, + "58": 1439181312.0, + "59": 1439181312.0, + "60": 1439181312.0, + "61": 1439181312.0, + "62": 1439181312.0, + "63": 1439181312.0, + "64": 1439181312.0, + "65": 1439181312.0, + "66": 1439181312.0, + "67": 1439181312.0, + "68": 1439181312.0, + "69": 1439181312.0, + "70": 1439181312.0, + "71": 1439181312.0, + "72": 1439181312.0, + "73": 1439181312.0, + "74": 1439181312.0, + "75": 1439181312.0, + "76": 1439181312.0, + "77": 1439181312.0, + "78": 1439181312.0, + "79": 1439181312.0, + "80": 1439181312.0, + "81": 1439181312.0, + "82": 1439181312.0, + "83": 1439181312.0, + "84": 1439181312.0, + "85": 1439181312.0, + "86": 1439181312.0, + "87": 1439181312.0, + "88": 1439181312.0, + "89": 1439181312.0, + "90": 1439181312.0, + "91": 1439181312.0, + "92": 1439181312.0, + "93": 1439181312.0, + "94": 1439181312.0, + "95": 1439181312.0, + "96": 1439181312.0, + "97": 1439181312.0, + "98": 1439181312.0, + "99": 1439181312.0, + "100": 1439181312.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.58245, - "3": 0.21353, - "4": 0.19809, - "5": 0.19973, - "6": 0.20032, - "7": 0.20139, - "8": 0.20295, - "9": 0.20043, - "10": 0.19972, - "11": 0.20132, - "12": 0.19792, - "13": 0.19682, - "14": 0.19821, - "15": 0.19848, - "16": 0.19746, - "17": 0.19514, - "18": 0.19827, - "19": 0.20219, - "20": 0.19693, - "21": 0.19737, - "22": 0.19747, - "23": 0.19919, - "24": 0.19926, - "25": 0.20085, - "26": 0.19991, - "27": 0.1997, - "28": 0.20048, - "29": 0.1998, - "30": 0.20018, - "31": 0.19925, - "32": 0.20106, - "33": 0.20211, - "34": 0.20138, - "35": 0.20509, - "36": 0.20306, - "37": 0.20105, - "38": 0.20493, - "39": 0.20228, - "40": 0.20024, - "41": 0.19764, - "42": 0.19823, - "43": 0.19904, - "44": 0.19748, - "45": 0.19769, - "46": 0.19962, - "47": 0.19909, - "48": 0.19707, - "49": 0.19811, - "50": 0.19797, - "51": 0.30241, - "52": 0.23416, - "53": 0.218, - "54": 0.24033, - "55": 0.20431, - "56": 0.20021, - "57": 0.19797, - "58": 0.19728, - "59": 0.19755, - "60": 0.19945, - "61": 0.19937, - "62": 0.20023, - "63": 0.19834, - "64": 0.19772, - "65": 0.19919, - "66": 0.19937, - "67": 0.1968, - "68": 0.19708, - "69": 0.19625, - "70": 0.19907, - "71": 0.19787, - "72": 0.19669, - "73": 0.1974, - "74": 0.19762, - "75": 0.19809, - "76": 0.19777, - "77": 0.19682, - "78": 0.19775, - "79": 0.19837, - "80": 0.20058, - "81": 0.19751, - "82": 0.19879, - "83": 0.19845, - "84": 0.19768, - "85": 0.19836, - "86": 0.19995, - "87": 0.19895, - "88": 0.1986, - "89": 0.20173, - "90": 0.2002, - "91": 0.19987, - "92": 0.19735, - "93": 0.1975, - "94": 0.19888, - "95": 0.19837, - "96": 0.19849, - "97": 0.19877, - "98": 0.19578, - "99": 0.19678, - "100": 0.19821 + "2": 7.28701, + "3": 0.22788, + "4": 0.17667, + "5": 0.17718, + "6": 0.17304, + "7": 0.17733, + "8": 0.1754, + "9": 0.17925, + "10": 0.1763, + "11": 0.17271, + "12": 0.21973, + "13": 0.17339, + "14": 0.17372, + "15": 0.17684, + "16": 0.17967, + "17": 0.1774, + "18": 0.17694, + "19": 0.17736, + "20": 0.17947, + "21": 0.17847, + "22": 0.17828, + "23": 0.17783, + "24": 0.17871, + "25": 0.17561, + "26": 0.20638, + "27": 0.32784, + "28": 0.30392, + "29": 0.30928, + "30": 0.35118, + "31": 0.27213, + "32": 0.17817, + "33": 0.17978, + "34": 0.1771, + "35": 0.17657, + "36": 0.17516, + "37": 0.17661, + "38": 0.17575, + "39": 0.17702, + "40": 0.17653, + "41": 0.17935, + "42": 0.17754, + "43": 0.17608, + "44": 0.1766, + "45": 0.17832, + "46": 0.17586, + "47": 0.17463, + "48": 0.17729, + "49": 0.17667, + "50": 0.17842, + "51": 0.31463, + "52": 0.23177, + "53": 0.1823, + "54": 0.17646, + "55": 0.17736, + "56": 0.17611, + "57": 0.17485, + "58": 0.1745, + "59": 0.23643, + "60": 0.17421, + "61": 0.17603, + "62": 0.17614, + "63": 0.17401, + "64": 0.17479, + "65": 0.17442, + "66": 0.17631, + "67": 0.18225, + "68": 0.17658, + "69": 0.17551, + "70": 0.17512, + "71": 0.1771, + "72": 0.17549, + "73": 0.17423, + "74": 0.17483, + "75": 0.17559, + "76": 0.17559, + "77": 0.17576, + "78": 0.22183, + "79": 0.17355, + "80": 0.17413, + "81": 0.17613, + "82": 0.17746, + "83": 0.17537, + "84": 0.17542, + "85": 0.17576, + "86": 0.17551, + "87": 0.17609, + "88": 0.17634, + "89": 0.17523, + "90": 0.17388, + "91": 0.17376, + "92": 0.17821, + "93": 0.17558, + "94": 0.17392, + "95": 0.17488, + "96": 0.17449, + "97": 0.17409, + "98": 0.17441, + "99": 0.17516, + "100": 0.17608 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json index e02afd01b1f..fa14a156c22 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, - "3": 10.94862, + "1": 10.95177, + "2": 10.94867, + "3": 10.94863, "4": 10.94969, - "5": 10.94753, - "6": 10.94472, + "5": 10.94755, + "6": 10.94479, "7": 10.9465, - "8": 10.94472, - "9": 10.94186, - "10": 10.94475, + "8": 10.94479, + "9": 10.94191, + "10": 10.94479, "11": 10.94106, - "12": 10.93297, + "12": 10.93291, "13": 10.92308, - "14": 10.92593, - "15": 10.89969, - "16": 10.89084, - "17": 10.89814, - "18": 10.88337, - "19": 10.88981, - "20": 10.79731, - "21": 10.79299, - "22": 10.77996, - "23": 10.76964, - "24": 10.74573, - "25": 10.73536, - "26": 10.72512, - "27": 10.67795, - "28": 10.62002, - "29": 10.59158, - "30": 10.56328, - "31": 10.56514, - "32": 10.54392, - "33": 10.50914, - "34": 10.48321, - "35": 10.46994, - "36": 10.44683, - "37": 10.42648, - "38": 10.43048, - "39": 10.39271, - "40": 10.38041, - "41": 10.35136, - "42": 10.3275, + "14": 10.92589, + "15": 10.89963, + "16": 10.89081, + "17": 10.89822, + "18": 10.88344, + "19": 10.88987, + "20": 10.7974, + "21": 10.79296, + "22": 10.78008, + "23": 10.76968, + "24": 10.74582, + "25": 10.73534, + "26": 10.72506, + "27": 10.67793, + "28": 10.62001, + "29": 10.59163, + "30": 10.56335, + "31": 10.56519, + "32": 10.544, + "33": 10.50913, + "34": 10.48323, + "35": 10.4699, + "36": 10.44694, + "37": 10.42657, + "38": 10.43051, + "39": 10.39275, + "40": 10.3804, + "41": 10.35141, + "42": 10.32755, "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24783, - "47": 10.2438, + "44": 10.28348, + "45": 10.29608, + "46": 10.24785, + "47": 10.24385, "48": 10.19393, - "49": 10.19673, - "50": 10.18924, - "51": 10.19745, + "49": 10.19678, + "50": 10.18927, + "51": 10.19748, "52": 10.15223, - "53": 10.15768, - "54": 10.12071, - "55": 10.09673, - "56": 10.12108, - "57": 10.11169, - "58": 10.12097, - "59": 10.06243, - "60": 10.09231, - "61": 10.04173, - "62": 10.00957, - "63": 10.07951, - "64": 10.03237, - "65": 10.00321, - "66": 10.03953, - "67": 10.01958, - "68": 9.9857, - "69": 10.00195, + "53": 10.15775, + "54": 10.12079, + "55": 10.09676, + "56": 10.1211, + "57": 10.11179, + "58": 10.12101, + "59": 10.06249, + "60": 10.0924, + "61": 10.04177, + "62": 10.00959, + "63": 10.0795, + "64": 10.03239, + "65": 10.00322, + "66": 10.03956, + "67": 10.01964, + "68": 9.98577, + "69": 10.00204, "70": 9.98144, - "71": 10.00456, - "72": 9.99623, - "73": 9.9876, - "74": 9.97288, - "75": 9.93232, - "76": 9.96721, - "77": 9.96946, - "78": 9.92083, - "79": 9.91736, - "80": 9.93738, - "81": 9.95721, - "82": 9.89591, - "83": 9.86219, - "84": 9.80485, - "85": 9.78642, - "86": 9.89557, - "87": 9.90872, + "71": 10.00455, + "72": 9.99626, + "73": 9.98765, + "74": 9.97294, + "75": 9.93237, + "76": 9.9672, + "77": 9.96954, + "78": 9.92086, + "79": 9.9174, + "80": 9.93744, + "81": 9.95731, + "82": 9.89594, + "83": 9.86223, + "84": 9.80486, + "85": 9.78646, + "86": 9.89559, + "87": 9.90877, "88": 9.88858, - "89": 9.83328, - "90": 9.82422, - "91": 9.8393, - "92": 9.82245, - "93": 9.75866, - "94": 9.83495, - "95": 9.82146, - "96": 9.81157, - "97": 9.75625, - "98": 9.78244, - "99": 9.82335, - "100": 9.71583 + "89": 9.8333, + "90": 9.82427, + "91": 9.83932, + "92": 9.82252, + "93": 9.75876, + "94": 9.83501, + "95": 9.82152, + "96": 9.81164, + "97": 9.75629, + "98": 9.78246, + "99": 9.82339, + "100": 9.71587 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1733.0, - "2": 1654.0, - "3": 1807.0, - "4": 1633.0, - "5": 1831.0, - "6": 1678.0, - "7": 1895.0, - "8": 1671.0, - "9": 1673.0, - "10": 1708.0, - "11": 1553.0, - "12": 1673.0, - "13": 1817.0, - "14": 1834.0, - "15": 1646.0, - "16": 1768.0, - "17": 1815.0, - "18": 1755.0, - "19": 1673.0, - "20": 1652.0, - "21": 1774.0, - "22": 1666.0, - "23": 1742.0, - "24": 1813.0, - "25": 1613.0, - "26": 1757.0, - "27": 1933.0, - "28": 1878.0, - "29": 1897.0, - "30": 1773.0, - "31": 2012.0, - "32": 1945.0, - "33": 1954.0, - "34": 2032.0, - "35": 2079.0, - "36": 1910.0, - "37": 2215.0, - "38": 2091.0, - "39": 2118.0, - "40": 2207.0, - "41": 2215.0, - "42": 2146.0, - "43": 2268.0, - "44": 2277.0, - "45": 2471.0, - "46": 2361.0, - "47": 2478.0, - "48": 2516.0, - "49": 2742.0, - "50": 2642.0, - "51": 2514.0, - "52": 2737.0, - "53": 2633.0, - "54": 2812.0, - "55": 2551.0, - "56": 2785.0, - "57": 2305.0, - "58": 3578.0, - "59": 2929.0, - "60": 2969.0, - "61": 2767.0, - "62": 3207.0, - "63": 3261.0, - "64": 3564.0, - "65": 2658.0, - "66": 3028.0, - "67": 3825.0, - "68": 3302.0, - "69": 2965.0, - "70": 3273.0, - "71": 3178.0, - "72": 2916.0, - "73": 3501.0, - "74": 3278.0, - "75": 3294.0, - "76": 3359.0, - "77": 3682.0, - "78": 3230.0, - "79": 3223.0, - "80": 2932.0, - "81": 3215.0, - "82": 2894.0, - "83": 2944.0, - "84": 3031.0, - "85": 2667.0, - "86": 3211.0, - "87": 2910.0, - "88": 3105.0, - "89": 2985.0, - "90": 3930.0, - "91": 2757.0, - "92": 3004.0, - "93": 2976.0, - "94": 3091.0, - "95": 3323.0, - "96": 3464.0, - "97": 3570.0, - "98": 3215.0, - "99": 3459.0, - "100": 3386.0 + "1": 1720.0, + "2": 1610.0, + "3": 1759.0, + "4": 1672.0, + "5": 1790.0, + "6": 1645.0, + "7": 1951.0, + "8": 1673.0, + "9": 1638.0, + "10": 1711.0, + "11": 1665.0, + "12": 1693.0, + "13": 1756.0, + "14": 1856.0, + "15": 1588.0, + "16": 1726.0, + "17": 1789.0, + "18": 1794.0, + "19": 1682.0, + "20": 1650.0, + "21": 1797.0, + "22": 1651.0, + "23": 1734.0, + "24": 1800.0, + "25": 1706.0, + "26": 1766.0, + "27": 1833.0, + "28": 1796.0, + "29": 1794.0, + "30": 1818.0, + "31": 1985.0, + "32": 1935.0, + "33": 1949.0, + "34": 2021.0, + "35": 2052.0, + "36": 2020.0, + "37": 2106.0, + "38": 2035.0, + "39": 2189.0, + "40": 2178.0, + "41": 2319.0, + "42": 2030.0, + "43": 2423.0, + "44": 2211.0, + "45": 2387.0, + "46": 2376.0, + "47": 2480.0, + "48": 2587.0, + "49": 2711.0, + "50": 2618.0, + "51": 2475.0, + "52": 2747.0, + "53": 2656.0, + "54": 2815.0, + "55": 2536.0, + "56": 2716.0, + "57": 2263.0, + "58": 3635.0, + "59": 3047.0, + "60": 3031.0, + "61": 2806.0, + "62": 3246.0, + "63": 3289.0, + "64": 3593.0, + "65": 2540.0, + "66": 2944.0, + "67": 3836.0, + "68": 3485.0, + "69": 2941.0, + "70": 3355.0, + "71": 3196.0, + "72": 2909.0, + "73": 3498.0, + "74": 3224.0, + "75": 3120.0, + "76": 3269.0, + "77": 3648.0, + "78": 3293.0, + "79": 3169.0, + "80": 3022.0, + "81": 3305.0, + "82": 2934.0, + "83": 3071.0, + "84": 3060.0, + "85": 2653.0, + "86": 3079.0, + "87": 2941.0, + "88": 3158.0, + "89": 3045.0, + "90": 3795.0, + "91": 2844.0, + "92": 2887.0, + "93": 2965.0, + "94": 2974.0, + "95": 3328.0, + "96": 3516.0, + "97": 3595.0, + "98": 3304.0, + "99": 3431.0, + "100": 3362.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.8204, - "3": 0.1796, - "4": 0.17955, - "5": 0.17744, - "6": 0.17714, - "7": 0.1765, - "8": 0.17799, - "9": 0.176, - "10": 0.1772, - "11": 0.17674, - "12": 0.17776, - "13": 0.17897, - "14": 0.1768, - "15": 0.17847, - "16": 0.17939, - "17": 0.17937, - "18": 0.17718, - "19": 0.17568, - "20": 0.17745, - "21": 0.17946, - "22": 0.17801, - "23": 0.17828, - "24": 0.178, - "25": 0.17664, - "26": 0.17724, - "27": 0.17888, - "28": 0.17787, - "29": 0.17973, - "30": 0.17772, - "31": 0.17599, - "32": 0.17666, - "33": 0.17886, - "34": 0.17748, - "35": 0.17691, - "36": 0.17764, - "37": 0.17648, - "38": 0.17664, - "39": 0.17676, - "40": 0.17609, - "41": 0.17722, - "42": 0.1764, - "43": 0.1777, - "44": 0.17703, - "45": 0.17688, - "46": 0.17623, - "47": 0.17635, - "48": 0.17849, - "49": 0.17982, - "50": 0.17674, - "51": 0.22322, - "52": 0.2255, - "53": 0.18599, - "54": 0.18169, - "55": 0.18182, - "56": 0.18479, - "57": 0.1827, - "58": 0.18224, - "59": 0.1835, - "60": 0.18081, - "61": 0.18445, - "62": 0.18249, - "63": 0.18229, - "64": 0.18312, - "65": 0.18169, - "66": 0.18557, - "67": 0.18168, - "68": 0.18285, - "69": 0.18255, - "70": 0.18126, - "71": 0.18048, - "72": 0.18212, - "73": 0.18187, - "74": 0.18287, - "75": 0.18889, - "76": 0.1857, - "77": 0.1841, - "78": 0.18312, - "79": 0.18172, - "80": 0.18173, - "81": 0.18191, - "82": 0.18203, - "83": 0.18296, - "84": 0.18092, - "85": 0.1841, - "86": 0.1816, - "87": 0.18873, - "88": 0.1838, - "89": 0.18246, - "90": 0.18023, - "91": 0.18356, - "92": 0.18142, - "93": 0.18856, - "94": 0.18076, - "95": 0.17997, - "96": 0.18093, - "97": 0.18031, - "98": 0.18139, - "99": 0.18029, - "100": 0.17496 + "2": 5.07529, + "3": 0.18885, + "4": 0.19106, + "5": 0.19207, + "6": 0.19245, + "7": 0.19051, + "8": 0.1888, + "9": 0.18878, + "10": 0.19262, + "11": 0.19083, + "12": 0.1888, + "13": 0.19172, + "14": 0.19068, + "15": 0.19093, + "16": 0.19383, + "17": 0.19015, + "18": 0.19139, + "19": 0.1913, + "20": 0.18797, + "21": 0.19207, + "22": 0.18992, + "23": 0.18989, + "24": 0.18947, + "25": 0.18998, + "26": 0.1899, + "27": 0.18841, + "28": 0.18949, + "29": 0.18881, + "30": 0.1886, + "31": 0.18532, + "32": 0.18839, + "33": 0.18929, + "34": 0.18788, + "35": 0.18782, + "36": 0.18724, + "37": 0.19267, + "38": 0.18847, + "39": 0.18774, + "40": 0.18826, + "41": 0.18816, + "42": 0.18659, + "43": 0.18882, + "44": 0.19062, + "45": 0.18783, + "46": 0.19262, + "47": 0.18993, + "48": 0.18827, + "49": 0.19113, + "50": 0.18973, + "51": 0.22535, + "52": 0.23171, + "53": 0.19289, + "54": 0.20035, + "55": 0.18976, + "56": 0.20079, + "57": 0.19722, + "58": 0.20346, + "59": 0.2038, + "60": 0.19645, + "61": 0.19749, + "62": 0.21023, + "63": 0.20154, + "64": 0.20037, + "65": 0.19468, + "66": 0.20083, + "67": 0.19723, + "68": 0.19807, + "69": 0.19466, + "70": 0.19516, + "71": 0.19117, + "72": 0.18882, + "73": 0.18964, + "74": 0.19736, + "75": 0.19688, + "76": 0.2106, + "77": 0.19209, + "78": 0.20181, + "79": 0.19509, + "80": 0.18984, + "81": 0.19824, + "82": 0.19122, + "83": 0.19711, + "84": 0.19525, + "85": 0.20466, + "86": 0.19019, + "87": 0.20727, + "88": 0.19502, + "89": 0.19972, + "90": 0.19036, + "91": 0.19535, + "92": 0.20058, + "93": 0.20625, + "94": 0.19663, + "95": 0.19694, + "96": 0.19008, + "97": 0.18989, + "98": 0.19735, + "99": 0.19873, + "100": 0.19488 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json index 26b1b73478d..025f6e42a8c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92866, - "4": 10.9288, - "5": 10.93088, - "6": 10.92771, - "7": 10.92336, - "8": 10.92159, - "9": 10.92031, + "2": 10.92832, + "3": 10.92864, + "4": 10.92888, + "5": 10.93089, + "6": 10.92772, + "7": 10.92341, + "8": 10.92161, + "9": 10.92027, "10": 10.91964, - "11": 10.91027, - "12": 10.91814, - "13": 10.91523, - "14": 10.89825, - "15": 10.88042, - "16": 10.86841, + "11": 10.91024, + "12": 10.91826, + "13": 10.91526, + "14": 10.89832, + "15": 10.88051, + "16": 10.8684, "17": 10.86882, - "18": 10.86258, - "19": 10.86185, - "20": 10.77348, - "21": 10.75516, - "22": 10.75521, - "23": 10.74508, - "24": 10.71106, - "25": 10.71082, - "26": 10.69968, - "27": 10.65192, - "28": 10.58249, - "29": 10.55636, - "30": 10.54863, - "31": 10.52499, - "32": 10.51761, - "33": 10.48144, - "34": 10.451, - "35": 10.44994, - "36": 10.43486, - "37": 10.39411, - "38": 10.39546, - "39": 10.36016, - "40": 10.35798, - "41": 10.32782, - "42": 10.30704, - "43": 10.28206, - "44": 10.25604, - "45": 10.27039, - "46": 10.23404, - "47": 10.21882, + "18": 10.86249, + "19": 10.86187, + "20": 10.77339, + "21": 10.75518, + "22": 10.75531, + "23": 10.74509, + "24": 10.71112, + "25": 10.71086, + "26": 10.69962, + "27": 10.65199, + "28": 10.58247, + "29": 10.55645, + "30": 10.54865, + "31": 10.52506, + "32": 10.51769, + "33": 10.48149, + "34": 10.45111, + "35": 10.44997, + "36": 10.43493, + "37": 10.3941, + "38": 10.39549, + "39": 10.36015, + "40": 10.35806, + "41": 10.32785, + "42": 10.30705, + "43": 10.28217, + "44": 10.25607, + "45": 10.27045, + "46": 10.23413, + "47": 10.21893, "48": 10.17023, - "49": 10.1759, - "50": 10.17855, - "51": 10.19228, - "52": 10.13723, - "53": 10.13552, - "54": 10.10521, - "55": 10.08055, - "56": 10.11116, - "57": 10.09599, - "58": 10.10931, - "59": 10.05623, - "60": 10.07399, - "61": 10.02775, - "62": 10.0003, - "63": 10.07322, - "64": 10.03695, - "65": 10.00341, - "66": 10.03038, - "67": 10.0072, - "68": 9.97019, - "69": 9.9915, - "70": 9.97453, - "71": 10.00065, - "72": 9.97779, - "73": 9.97166, - "74": 9.95604, - "75": 9.93149, - "76": 9.96344, - "77": 9.95639, - "78": 9.90552, - "79": 9.91218, - "80": 9.92898, - "81": 9.95655, - "82": 9.89202, - "83": 9.85589, - "84": 9.78695, - "85": 9.78572, - "86": 9.88392, - "87": 9.91028, - "88": 9.88719, - "89": 9.81819, - "90": 9.81249, - "91": 9.82779, - "92": 9.81659, - "93": 9.74998, - "94": 9.8309, - "95": 9.82648, - "96": 9.80702, - "97": 9.74393, - "98": 9.77873, - "99": 9.82058, - "100": 9.71398 + "49": 10.17594, + "50": 10.17859, + "51": 10.19235, + "52": 10.13725, + "53": 10.13558, + "54": 10.10524, + "55": 10.08058, + "56": 10.11123, + "57": 10.09601, + "58": 10.10934, + "59": 10.05632, + "60": 10.07401, + "61": 10.02777, + "62": 10.00036, + "63": 10.07317, + "64": 10.03696, + "65": 10.00345, + "66": 10.0304, + "67": 10.00724, + "68": 9.97024, + "69": 9.99153, + "70": 9.9746, + "71": 10.00066, + "72": 9.97785, + "73": 9.9717, + "74": 9.95607, + "75": 9.93157, + "76": 9.96345, + "77": 9.95644, + "78": 9.90557, + "79": 9.9122, + "80": 9.92906, + "81": 9.95666, + "82": 9.89201, + "83": 9.85599, + "84": 9.78701, + "85": 9.7858, + "86": 9.88398, + "87": 9.91034, + "88": 9.88728, + "89": 9.81818, + "90": 9.81251, + "91": 9.82784, + "92": 9.81667, + "93": 9.75004, + "94": 9.83094, + "95": 9.82651, + "96": 9.80705, + "97": 9.74396, + "98": 9.77876, + "99": 9.82056, + "100": 9.71405 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1731.0, - "2": 1691.0, - "3": 1669.0, - "4": 1581.0, - "5": 1644.0, - "6": 1685.0, - "7": 1815.0, - "8": 1709.0, - "9": 1781.0, - "10": 1748.0, - "11": 1630.0, - "12": 1732.0, - "13": 1742.0, - "14": 1815.0, - "15": 1549.0, - "16": 1654.0, - "17": 1749.0, - "18": 1742.0, - "19": 1686.0, - "20": 1643.0, - "21": 1695.0, - "22": 1733.0, - "23": 1658.0, - "24": 1781.0, - "25": 1666.0, - "26": 1789.0, - "27": 1777.0, - "28": 1803.0, - "29": 1831.0, - "30": 1831.0, - "31": 2000.0, - "32": 1953.0, - "33": 1927.0, - "34": 2012.0, - "35": 2071.0, - "36": 1876.0, - "37": 2178.0, - "38": 2133.0, - "39": 2155.0, - "40": 2357.0, - "41": 2339.0, - "42": 1887.0, - "43": 2351.0, - "44": 2221.0, - "45": 2475.0, - "46": 2480.0, - "47": 2497.0, + "1": 1684.0, + "2": 1628.0, + "3": 1753.0, + "4": 1661.0, + "5": 1709.0, + "6": 1619.0, + "7": 1824.0, + "8": 1706.0, + "9": 1707.0, + "10": 1648.0, + "11": 1619.0, + "12": 1773.0, + "13": 1782.0, + "14": 1928.0, + "15": 1597.0, + "16": 1623.0, + "17": 1698.0, + "18": 1750.0, + "19": 1626.0, + "20": 1618.0, + "21": 1771.0, + "22": 1706.0, + "23": 1626.0, + "24": 1718.0, + "25": 1588.0, + "26": 1757.0, + "27": 1832.0, + "28": 1744.0, + "29": 1941.0, + "30": 1853.0, + "31": 1889.0, + "32": 1871.0, + "33": 1977.0, + "34": 2042.0, + "35": 2009.0, + "36": 1943.0, + "37": 2226.0, + "38": 2146.0, + "39": 2107.0, + "40": 2290.0, + "41": 2277.0, + "42": 1917.0, + "43": 2428.0, + "44": 2234.0, + "45": 2588.0, + "46": 2488.0, + "47": 2491.0, "48": 2601.0, - "49": 2909.0, - "50": 2638.0, - "51": 2495.0, - "52": 2790.0, - "53": 2729.0, - "54": 2777.0, - "55": 2626.0, - "56": 2743.0, - "57": 2177.0, - "58": 3682.0, - "59": 2899.0, - "60": 2899.0, - "61": 2826.0, - "62": 3369.0, - "63": 3377.0, - "64": 3714.0, - "65": 2750.0, - "66": 3201.0, - "67": 3778.0, - "68": 3542.0, - "69": 3015.0, - "70": 3264.0, - "71": 3129.0, - "72": 3054.0, - "73": 3365.0, - "74": 3234.0, - "75": 3345.0, - "76": 3367.0, - "77": 3958.0, - "78": 3329.0, - "79": 3194.0, - "80": 2951.0, - "81": 3592.0, - "82": 2943.0, - "83": 3073.0, - "84": 3037.0, - "85": 2541.0, - "86": 3023.0, - "87": 2916.0, - "88": 3027.0, - "89": 3180.0, - "90": 3778.0, - "91": 3014.0, - "92": 2726.0, - "93": 3051.0, - "94": 3000.0, - "95": 3189.0, - "96": 3382.0, - "97": 3490.0, - "98": 3305.0, - "99": 3298.0, - "100": 3311.0 + "49": 2860.0, + "50": 2622.0, + "51": 2566.0, + "52": 2713.0, + "53": 2694.0, + "54": 2837.0, + "55": 2660.0, + "56": 2691.0, + "57": 2100.0, + "58": 3764.0, + "59": 2897.0, + "60": 3048.0, + "61": 2813.0, + "62": 3200.0, + "63": 3426.0, + "64": 3603.0, + "65": 2607.0, + "66": 3090.0, + "67": 3762.0, + "68": 3476.0, + "69": 3189.0, + "70": 3310.0, + "71": 3107.0, + "72": 3040.0, + "73": 3459.0, + "74": 3262.0, + "75": 3167.0, + "76": 3312.0, + "77": 3909.0, + "78": 3290.0, + "79": 3200.0, + "80": 3133.0, + "81": 3597.0, + "82": 2947.0, + "83": 3172.0, + "84": 2917.0, + "85": 2518.0, + "86": 3140.0, + "87": 2832.0, + "88": 3076.0, + "89": 3067.0, + "90": 3800.0, + "91": 3056.0, + "92": 2804.0, + "93": 3105.0, + "94": 3146.0, + "95": 3205.0, + "96": 3332.0, + "97": 3391.0, + "98": 3361.0, + "99": 3318.0, + "100": 3245.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.02392, - "3": 2.50573, - "4": 2.4125, - "5": 2.18239, - "6": 2.29587, - "7": 1.85413, - "8": 2.68302, - "9": 2.677, - "10": 2.76099, - "11": 2.05385, - "12": 1.68068, - "13": 1.92045, - "14": 2.56065, - "15": 1.90139, - "16": 1.57226, - "17": 2.41383, - "18": 2.14416, - "19": 2.05606, - "20": 2.16357, - "21": 2.13152, - "22": 1.50502, - "23": 2.08983, - "24": 1.63963, - "25": 2.29431, - "26": 2.83751, - "27": 1.76414, - "28": 2.39332, - "29": 1.62607, - "30": 1.67187, - "31": 1.6553, - "32": 2.55786, - "33": 2.06527, - "34": 2.06572, - "35": 2.81203, - "36": 2.55928, - "37": 2.06661, - "38": 1.38414, - "39": 3.16205, - "40": 1.82841, - "41": 2.00472, - "42": 2.51756, - "43": 1.66235, - "44": 1.38274, - "45": 2.299, - "46": 1.55558, - "47": 2.08796, - "48": 1.98617, - "49": 1.72974, - "50": 2.01108, - "51": 1.56822, - "52": 2.26704, - "53": 2.26243, - "54": 2.24808, - "55": 1.79302, - "56": 2.73831, - "57": 0.92359, - "58": 2.06055, - "59": 1.75395, - "60": 2.24972, - "61": 3.25644, - "62": 3.72146, - "63": 2.11213, - "64": 2.34726, - "65": 2.21863, - "66": 2.96183, - "67": 1.90196, - "68": 1.55317, - "69": 1.46562, - "70": 1.99154, - "71": 2.26399, - "72": 1.9531, - "73": 1.92068, - "74": 1.80936, - "75": 1.6175, - "76": 1.75104, - "77": 1.5674, - "78": 2.05942, - "79": 2.19012, - "80": 2.1471, - "81": 2.4255, - "82": 2.30913, - "83": 2.09438, - "84": 1.89029, - "85": 1.79793, - "86": 2.03875, - "87": 2.23952, - "88": 2.10309, - "89": 2.395, - "90": 1.68837, - "91": 2.3603, - "92": 2.54642, - "93": 1.76393, - "94": 2.36051, - "95": 1.82959, - "96": 1.69184, - "97": 1.76942, - "98": 2.12513, - "99": 2.47915, - "100": 1.81358 + "2": 4.76237, + "3": 1.75706, + "4": 1.60852, + "5": 1.17428, + "6": 1.41718, + "7": 1.22496, + "8": 1.8776, + "9": 1.78166, + "10": 1.81017, + "11": 1.74527, + "12": 1.77374, + "13": 1.8609, + "14": 1.86026, + "15": 1.47495, + "16": 1.57228, + "17": 2.0464, + "18": 1.25244, + "19": 1.52828, + "20": 1.56834, + "21": 1.71348, + "22": 1.27694, + "23": 1.71413, + "24": 1.20187, + "25": 1.6052, + "26": 1.79244, + "27": 1.64653, + "28": 1.30306, + "29": 1.21507, + "30": 1.47371, + "31": 1.18905, + "32": 2.0757, + "33": 1.36838, + "34": 1.28116, + "35": 1.79996, + "36": 1.55778, + "37": 1.85066, + "38": 1.20291, + "39": 2.019, + "40": 1.53632, + "41": 1.72081, + "42": 2.30246, + "43": 1.51592, + "44": 1.46633, + "45": 1.84999, + "46": 1.69297, + "47": 1.58387, + "48": 2.05203, + "49": 1.32292, + "50": 1.86151, + "51": 1.56381, + "52": 1.99421, + "53": 1.8652, + "54": 2.08428, + "55": 1.81562, + "56": 2.72064, + "57": 1.49666, + "58": 2.35577, + "59": 1.76845, + "60": 1.59287, + "61": 1.61297, + "62": 1.91264, + "63": 1.51233, + "64": 1.84452, + "65": 1.62218, + "66": 1.76331, + "67": 1.83044, + "68": 1.47005, + "69": 1.74338, + "70": 1.67262, + "71": 2.37194, + "72": 1.5982, + "73": 1.61342, + "74": 1.67804, + "75": 1.67445, + "76": 1.63682, + "77": 1.86252, + "78": 1.87517, + "79": 2.19764, + "80": 1.6041, + "81": 1.871, + "82": 2.23216, + "83": 1.77526, + "84": 1.46961, + "85": 1.53686, + "86": 1.67694, + "87": 2.36988, + "88": 2.07752, + "89": 2.14852, + "90": 1.9958, + "91": 1.97798, + "92": 2.40945, + "93": 1.62165, + "94": 1.87215, + "95": 1.86571, + "96": 1.43339, + "97": 1.42161, + "98": 1.86086, + "99": 2.40818, + "100": 1.74417 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json index c56db7403ec..fb3ee151165 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92942, - "2": 10.92777, - "3": 10.92678, - "4": 10.92611, - "5": 10.9267, - "6": 10.92765, - "7": 10.92397, - "8": 10.91828, - "9": 10.91963, - "10": 10.91773, - "11": 10.90809, - "12": 10.91698, - "13": 10.91062, - "14": 10.90204, - "15": 10.88112, - "16": 10.87324, - "17": 10.86927, - "18": 10.86348, - "19": 10.86027, - "20": 10.77042, - "21": 10.75578, - "22": 10.7538, - "23": 10.7458, - "24": 10.71437, - "25": 10.71349, - "26": 10.70281, - "27": 10.65475, - "28": 10.57856, - "29": 10.55898, + "1": 10.92954, + "2": 10.92779, + "3": 10.92674, + "4": 10.92614, + "5": 10.92679, + "6": 10.92769, + "7": 10.92393, + "8": 10.91823, + "9": 10.91969, + "10": 10.91778, + "11": 10.90802, + "12": 10.91693, + "13": 10.91061, + "14": 10.90208, + "15": 10.88116, + "16": 10.87326, + "17": 10.86925, + "18": 10.86347, + "19": 10.8603, + "20": 10.77048, + "21": 10.75585, + "22": 10.75375, + "23": 10.74582, + "24": 10.71442, + "25": 10.71354, + "26": 10.70284, + "27": 10.65476, + "28": 10.57867, + "29": 10.55901, "30": 10.54748, - "31": 10.52975, - "32": 10.51717, - "33": 10.48071, - "34": 10.45115, - "35": 10.45045, - "36": 10.4332, - "37": 10.3935, - "38": 10.39542, - "39": 10.36365, - "40": 10.35668, - "41": 10.3257, - "42": 10.30768, - "43": 10.28178, - "44": 10.25424, - "45": 10.27022, - "46": 10.23516, - "47": 10.21978, + "31": 10.52977, + "32": 10.51724, + "33": 10.48072, + "34": 10.45122, + "35": 10.45046, + "36": 10.43323, + "37": 10.39355, + "38": 10.39544, + "39": 10.36366, + "40": 10.35675, + "41": 10.32574, + "42": 10.30776, + "43": 10.28186, + "44": 10.25433, + "45": 10.27029, + "46": 10.23523, + "47": 10.21981, "48": 10.16978, - "49": 10.1782, - "50": 10.17838, - "51": 10.19229, + "49": 10.17828, + "50": 10.17842, + "51": 10.19236, "52": 10.13724, - "53": 10.13535, - "54": 10.10652, - "55": 10.08095, - "56": 10.11299, - "57": 10.09527, - "58": 10.10883, - "59": 10.05599, - "60": 10.07525, - "61": 10.02735, - "62": 10.00229, - "63": 10.07215, - "64": 10.03494, - "65": 10.00276, - "66": 10.02917, - "67": 10.00665, - "68": 9.96971, - "69": 9.9904, - "70": 9.97336, - "71": 10.00126, - "72": 9.97761, - "73": 9.97165, - "74": 9.95655, - "75": 9.92901, - "76": 9.96292, + "53": 10.13538, + "54": 10.1066, + "55": 10.08098, + "56": 10.11294, + "57": 10.09536, + "58": 10.10886, + "59": 10.05607, + "60": 10.07526, + "61": 10.02741, + "62": 10.0023, + "63": 10.07218, + "64": 10.035, + "65": 10.00277, + "66": 10.02918, + "67": 10.0067, + "68": 9.96977, + "69": 9.99043, + "70": 9.97343, + "71": 10.0013, + "72": 9.97764, + "73": 9.97167, + "74": 9.95661, + "75": 9.92906, + "76": 9.96298, "77": 9.95725, - "78": 9.90597, - "79": 9.90995, - "80": 9.93063, - "81": 9.95608, - "82": 9.89121, - "83": 9.85515, + "78": 9.90604, + "79": 9.91006, + "80": 9.93069, + "81": 9.95614, + "82": 9.89127, + "83": 9.85523, "84": 9.78774, - "85": 9.78745, + "85": 9.78741, "86": 9.88334, - "87": 9.91031, - "88": 9.88594, - "89": 9.81827, - "90": 9.8142, - "91": 9.82559, + "87": 9.91032, + "88": 9.88596, + "89": 9.81828, + "90": 9.81422, + "91": 9.82561, "92": 9.81427, - "93": 9.75213, - "94": 9.83213, + "93": 9.7522, + "94": 9.83215, "95": 9.82479, - "96": 9.80678, - "97": 9.74325, - "98": 9.77879, - "99": 9.81994, - "100": 9.71218 + "96": 9.80688, + "97": 9.74331, + "98": 9.7788, + "99": 9.81996, + "100": 9.71224 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 66.0, - "2": 61.0, - "3": 49.0, - "4": 64.0, - "5": 65.0, - "6": 67.0, - "7": 69.0, + "1": 71.0, + "2": 59.0, + "3": 64.0, + "4": 78.0, + "5": 58.0, + "6": 50.0, + "7": 62.0, "8": 60.0, - "9": 51.0, - "10": 54.0, - "11": 58.0, - "12": 65.0, - "13": 57.0, - "14": 49.0, + "9": 67.0, + "10": 60.0, + "11": 68.0, + "12": 46.0, + "13": 64.0, + "14": 38.0, "15": 69.0, - "16": 56.0, - "17": 60.0, - "18": 66.0, - "19": 57.0, - "20": 50.0, - "21": 56.0, - "22": 59.0, - "23": 70.0, - "24": 65.0, - "25": 54.0, - "26": 73.0, - "27": 55.0, - "28": 75.0, - "29": 79.0, - "30": 74.0, - "31": 70.0, - "32": 80.0, - "33": 74.0, - "34": 76.0, - "35": 61.0, + "16": 60.0, + "17": 52.0, + "18": 64.0, + "19": 61.0, + "20": 59.0, + "21": 52.0, + "22": 80.0, + "23": 64.0, + "24": 74.0, + "25": 53.0, + "26": 64.0, + "27": 81.0, + "28": 66.0, + "29": 84.0, + "30": 66.0, + "31": 68.0, + "32": 65.0, + "33": 71.0, + "34": 71.0, + "35": 53.0, "36": 63.0, - "37": 91.0, - "38": 64.0, - "39": 68.0, - "40": 66.0, - "41": 81.0, - "42": 70.0, - "43": 82.0, - "44": 63.0, - "45": 77.0, - "46": 72.0, - "47": 85.0, - "48": 102.0, - "49": 94.0, - "50": 73.0, - "51": 61.0, - "52": 72.0, - "53": 82.0, - "54": 84.0, - "55": 74.0, - "56": 88.0, - "57": 80.0, - "58": 90.0, - "59": 83.0, - "60": 70.0, - "61": 82.0, - "62": 88.0, - "63": 69.0, - "64": 94.0, - "65": 98.0, - "66": 71.0, - "67": 97.0, - "68": 112.0, - "69": 96.0, - "70": 91.0, - "71": 75.0, - "72": 83.0, - "73": 91.0, - "74": 108.0, - "75": 78.0, - "76": 72.0, - "77": 79.0, - "78": 86.0, - "79": 91.0, - "80": 85.0, - "81": 101.0, - "82": 80.0, - "83": 95.0, - "84": 80.0, - "85": 75.0, - "86": 52.0, - "87": 85.0, - "88": 85.0, - "89": 76.0, - "90": 91.0, - "91": 76.0, - "92": 64.0, - "93": 55.0, - "94": 81.0, - "95": 72.0, - "96": 69.0, - "97": 93.0, - "98": 60.0, - "99": 94.0, - "100": 73.0 + "37": 62.0, + "38": 68.0, + "39": 74.0, + "40": 68.0, + "41": 65.0, + "42": 63.0, + "43": 69.0, + "44": 66.0, + "45": 87.0, + "46": 67.0, + "47": 66.0, + "48": 90.0, + "49": 97.0, + "50": 85.0, + "51": 82.0, + "52": 85.0, + "53": 75.0, + "54": 87.0, + "55": 93.0, + "56": 62.0, + "57": 63.0, + "58": 82.0, + "59": 69.0, + "60": 91.0, + "61": 81.0, + "62": 83.0, + "63": 81.0, + "64": 82.0, + "65": 81.0, + "66": 85.0, + "67": 93.0, + "68": 85.0, + "69": 91.0, + "70": 84.0, + "71": 69.0, + "72": 66.0, + "73": 83.0, + "74": 87.0, + "75": 80.0, + "76": 84.0, + "77": 95.0, + "78": 105.0, + "79": 82.0, + "80": 66.0, + "81": 87.0, + "82": 70.0, + "83": 93.0, + "84": 77.0, + "85": 60.0, + "86": 80.0, + "87": 74.0, + "88": 82.0, + "89": 88.0, + "90": 98.0, + "91": 77.0, + "92": 61.0, + "93": 71.0, + "94": 76.0, + "95": 79.0, + "96": 87.0, + "97": 72.0, + "98": 78.0, + "99": 98.0, + "100": 83.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 545423872.0, - "2": 545423872.0, - "3": 545423872.0, - "4": 545423872.0, - "5": 545423872.0, - "6": 545423872.0, - "7": 545423872.0, - "8": 545423872.0, - "9": 545423872.0, - "10": 545423872.0, - "11": 545423872.0, - "12": 545423872.0, - "13": 545423872.0, - "14": 545423872.0, - "15": 545423872.0, - "16": 545423872.0, - "17": 545423872.0, - "18": 545423872.0, - "19": 545423872.0, - "20": 545423872.0, - "21": 545423872.0, - "22": 545423872.0, - "23": 545423872.0, - "24": 545423872.0, - "25": 545423872.0, - "26": 545423872.0, - "27": 545423872.0, - "28": 545423872.0, - "29": 545423872.0, - "30": 545423872.0, - "31": 545423872.0, - "32": 545423872.0, - "33": 545423872.0, - "34": 545423872.0, - "35": 545423872.0, - "36": 545423872.0, - "37": 545423872.0, - "38": 545423872.0, - "39": 545423872.0, - "40": 545423872.0, - "41": 545423872.0, - "42": 545423872.0, - "43": 545423872.0, - "44": 545423872.0, - "45": 545423872.0, - "46": 545423872.0, - "47": 545423872.0, - "48": 545423872.0, - "49": 545423872.0, - "50": 545423872.0, - "51": 545423872.0, - "52": 545423872.0, - "53": 545423872.0, - "54": 545423872.0, - "55": 545423872.0, - "56": 545423872.0, - "57": 545423872.0, - "58": 545423872.0, - "59": 545423872.0, - "60": 545423872.0, - "61": 545423872.0, - "62": 545423872.0, - "63": 545423872.0, - "64": 545423872.0, - "65": 545423872.0, - "66": 545423872.0, - "67": 545423872.0, - "68": 545423872.0, - "69": 545423872.0, - "70": 545423872.0, - "71": 545423872.0, - "72": 545423872.0, - "73": 545423872.0, - "74": 545423872.0, - "75": 545423872.0, - "76": 545423872.0, - "77": 545423872.0, - "78": 545423872.0, - "79": 545423872.0, - "80": 545423872.0, - "81": 545423872.0, - "82": 545423872.0, - "83": 545423872.0, - "84": 545423872.0, - "85": 545423872.0, - "86": 545423872.0, - "87": 545423872.0, - "88": 545423872.0, - "89": 545423872.0, - "90": 545423872.0, - "91": 545423872.0, - "92": 545423872.0, - "93": 545423872.0, - "94": 545423872.0, - "95": 545423872.0, - "96": 545423872.0, - "97": 545423872.0, - "98": 545423872.0, - "99": 545423872.0, - "100": 545423872.0 + "1": 542802432.0, + "2": 542802432.0, + "3": 542802432.0, + "4": 542802432.0, + "5": 542802432.0, + "6": 542802432.0, + "7": 542802432.0, + "8": 542802432.0, + "9": 542802432.0, + "10": 542802432.0, + "11": 542802432.0, + "12": 542802432.0, + "13": 542802432.0, + "14": 542802432.0, + "15": 542802432.0, + "16": 542802432.0, + "17": 542802432.0, + "18": 542802432.0, + "19": 542802432.0, + "20": 542802432.0, + "21": 542802432.0, + "22": 542802432.0, + "23": 542802432.0, + "24": 542802432.0, + "25": 542802432.0, + "26": 542802432.0, + "27": 542802432.0, + "28": 542802432.0, + "29": 542802432.0, + "30": 542802432.0, + "31": 542802432.0, + "32": 542802432.0, + "33": 542802432.0, + "34": 542802432.0, + "35": 542802432.0, + "36": 542802432.0, + "37": 542802432.0, + "38": 542802432.0, + "39": 542802432.0, + "40": 542802432.0, + "41": 542802432.0, + "42": 542802432.0, + "43": 542802432.0, + "44": 542802432.0, + "45": 542802432.0, + "46": 542802432.0, + "47": 542802432.0, + "48": 542802432.0, + "49": 542802432.0, + "50": 542802432.0, + "51": 542802432.0, + "52": 542802432.0, + "53": 542802432.0, + "54": 542802432.0, + "55": 542802432.0, + "56": 542802432.0, + "57": 542802432.0, + "58": 542802432.0, + "59": 542802432.0, + "60": 542802432.0, + "61": 542802432.0, + "62": 542802432.0, + "63": 542802432.0, + "64": 542802432.0, + "65": 542802432.0, + "66": 542802432.0, + "67": 542802432.0, + "68": 542802432.0, + "69": 542802432.0, + "70": 542802432.0, + "71": 542802432.0, + "72": 542802432.0, + "73": 542802432.0, + "74": 542802432.0, + "75": 542802432.0, + "76": 542802432.0, + "77": 542802432.0, + "78": 542802432.0, + "79": 542802432.0, + "80": 542802432.0, + "81": 542802432.0, + "82": 542802432.0, + "83": 542802432.0, + "84": 542802432.0, + "85": 542802432.0, + "86": 542802432.0, + "87": 542802432.0, + "88": 542802432.0, + "89": 542802432.0, + "90": 542802432.0, + "91": 542802432.0, + "92": 542802432.0, + "93": 542802432.0, + "94": 542802432.0, + "95": 542802432.0, + "96": 542802432.0, + "97": 542802432.0, + "98": 542802432.0, + "99": 542802432.0, + "100": 542802432.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1726380544.0, - "2": 1906452992.0, - "3": 1906452992.0, - "4": 1906452992.0, - "5": 1906452992.0, - "6": 1906452992.0, - "7": 1906452992.0, - "8": 1906452992.0, - "9": 1906452992.0, - "10": 1906452992.0, - "11": 1906452992.0, - "12": 1906452992.0, - "13": 1906452992.0, - "14": 1906452992.0, - "15": 1906452992.0, - "16": 1906452992.0, - "17": 1906452992.0, - "18": 1906452992.0, - "19": 1906452992.0, - "20": 1906452992.0, - "21": 1906452992.0, - "22": 1906452992.0, - "23": 1906452992.0, - "24": 1906452992.0, - "25": 1906452992.0, - "26": 1906452992.0, - "27": 1906452992.0, - "28": 1906452992.0, - "29": 1906452992.0, - "30": 1906452992.0, - "31": 1906452992.0, - "32": 1906452992.0, - "33": 1906452992.0, - "34": 1906452992.0, - "35": 1906452992.0, - "36": 1906452992.0, - "37": 1906452992.0, - "38": 1906452992.0, - "39": 1906452992.0, - "40": 1906452992.0, - "41": 1906452992.0, - "42": 1906452992.0, - "43": 1906452992.0, - "44": 1906452992.0, - "45": 1906452992.0, - "46": 1906452992.0, - "47": 1906452992.0, - "48": 1906452992.0, - "49": 1906452992.0, - "50": 1906452992.0, - "51": 1907501568.0, - "52": 1907501568.0, - "53": 1907501568.0, - "54": 1907501568.0, - "55": 1907501568.0, - "56": 1907501568.0, - "57": 1907501568.0, - "58": 1907501568.0, - "59": 1907501568.0, - "60": 1907501568.0, - "61": 1907501568.0, - "62": 1907501568.0, - "63": 1907501568.0, - "64": 1907501568.0, - "65": 1907501568.0, - "66": 1907501568.0, - "67": 1907501568.0, - "68": 1907501568.0, - "69": 1907501568.0, - "70": 1907501568.0, - "71": 1907501568.0, - "72": 1907501568.0, - "73": 1907501568.0, - "74": 1907501568.0, - "75": 1907501568.0, - "76": 1907501568.0, - "77": 1907501568.0, - "78": 1907501568.0, - "79": 1907501568.0, - "80": 1907501568.0, - "81": 1907501568.0, - "82": 1907501568.0, - "83": 1907501568.0, - "84": 1907501568.0, - "85": 1907501568.0, - "86": 1907501568.0, - "87": 1907501568.0, - "88": 1907501568.0, - "89": 1907501568.0, - "90": 1907501568.0, - "91": 1907501568.0, - "92": 1907501568.0, - "93": 1907501568.0, - "94": 1907501568.0, - "95": 1907501568.0, - "96": 1907501568.0, - "97": 1907501568.0, - "98": 1907501568.0, - "99": 1907501568.0, - "100": 1907501568.0 + "2": 1905142272.0, + "3": 1905142272.0, + "4": 1905142272.0, + "5": 1905142272.0, + "6": 1905142272.0, + "7": 1905142272.0, + "8": 1905142272.0, + "9": 1905142272.0, + "10": 1905142272.0, + "11": 1905142272.0, + "12": 1905142272.0, + "13": 1905142272.0, + "14": 1905142272.0, + "15": 1905142272.0, + "16": 1905142272.0, + "17": 1905142272.0, + "18": 1905142272.0, + "19": 1905142272.0, + "20": 1905142272.0, + "21": 1905142272.0, + "22": 1905142272.0, + "23": 1905142272.0, + "24": 1905142272.0, + "25": 1905142272.0, + "26": 1905142272.0, + "27": 1905142272.0, + "28": 1905142272.0, + "29": 1905142272.0, + "30": 1905142272.0, + "31": 1905142272.0, + "32": 1905142272.0, + "33": 1905142272.0, + "34": 1905142272.0, + "35": 1905142272.0, + "36": 1905142272.0, + "37": 1905142272.0, + "38": 1905142272.0, + "39": 1905142272.0, + "40": 1905142272.0, + "41": 1905142272.0, + "42": 1905142272.0, + "43": 1905142272.0, + "44": 1905142272.0, + "45": 1905142272.0, + "46": 1905142272.0, + "47": 1905142272.0, + "48": 1905142272.0, + "49": 1905142272.0, + "50": 1905142272.0, + "51": 1905928704.0, + "52": 1905928704.0, + "53": 1905928704.0, + "54": 1905928704.0, + "55": 1905928704.0, + "56": 1905928704.0, + "57": 1905928704.0, + "58": 1905928704.0, + "59": 1905928704.0, + "60": 1905928704.0, + "61": 1905928704.0, + "62": 1905928704.0, + "63": 1905928704.0, + "64": 1905928704.0, + "65": 1905928704.0, + "66": 1905928704.0, + "67": 1905928704.0, + "68": 1905928704.0, + "69": 1905928704.0, + "70": 1905928704.0, + "71": 1905928704.0, + "72": 1905928704.0, + "73": 1905928704.0, + "74": 1905928704.0, + "75": 1905928704.0, + "76": 1905928704.0, + "77": 1905928704.0, + "78": 1905928704.0, + "79": 1905928704.0, + "80": 1905928704.0, + "81": 1905928704.0, + "82": 1905928704.0, + "83": 1905928704.0, + "84": 1905928704.0, + "85": 1905928704.0, + "86": 1905928704.0, + "87": 1905928704.0, + "88": 1905928704.0, + "89": 1905928704.0, + "90": 1905928704.0, + "91": 1905928704.0, + "92": 1905928704.0, + "93": 1905928704.0, + "94": 1905928704.0, + "95": 1905928704.0, + "96": 1905928704.0, + "97": 1905928704.0, + "98": 1905928704.0, + "99": 1905928704.0, + "100": 1905928704.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.73172, - "3": 0.21569, - "4": 0.19596, - "5": 0.19542, - "6": 0.1964, - "7": 0.19501, - "8": 0.19704, - "9": 0.19446, - "10": 0.19404, - "11": 0.19588, - "12": 0.1976, - "13": 0.19703, - "14": 0.19683, - "15": 0.19748, - "16": 0.19896, - "17": 0.19842, - "18": 0.19762, - "19": 0.20038, - "20": 0.19791, - "21": 0.20639, - "22": 0.2155, - "23": 0.19828, - "24": 0.1998, - "25": 0.19718, - "26": 0.1969, - "27": 0.19738, - "28": 0.19798, - "29": 0.19521, - "30": 0.19558, - "31": 0.19497, - "32": 0.19525, - "33": 0.19719, - "34": 0.1979, - "35": 0.19842, - "36": 0.19788, - "37": 0.19557, - "38": 0.19393, - "39": 0.19405, - "40": 0.19646, - "41": 0.19603, - "42": 0.19512, - "43": 0.19706, - "44": 0.19667, - "45": 0.19778, - "46": 0.1984, - "47": 0.1974, - "48": 0.19656, - "49": 0.19924, - "50": 0.19733, - "51": 0.32596, - "52": 0.25374, - "53": 0.20066, - "54": 0.19999, - "55": 0.19894, - "56": 0.19869, - "57": 0.1981, - "58": 0.19836, - "59": 0.19975, - "60": 0.19674, - "61": 0.1987, - "62": 0.19736, - "63": 0.19808, - "64": 0.19975, - "65": 0.20007, - "66": 0.19904, - "67": 0.19772, - "68": 0.19885, - "69": 0.19648, - "70": 0.19801, - "71": 0.19898, - "72": 0.1994, - "73": 0.21072, - "74": 0.24023, - "75": 0.24242, - "76": 0.23471, - "77": 0.21606, - "78": 0.19745, - "79": 0.21418, - "80": 0.23134, - "81": 0.21475, - "82": 0.21988, - "83": 0.19778, - "84": 0.19825, - "85": 0.19785, - "86": 0.19738, - "87": 0.19712, - "88": 0.19851, - "89": 0.19909, - "90": 0.19881, - "91": 0.19887, - "92": 0.19915, - "93": 0.19979, - "94": 0.19868, - "95": 0.1992, - "96": 0.19882, - "97": 0.19886, - "98": 0.19975, - "99": 0.20053, - "100": 0.20043 + "2": 7.47722, + "3": 0.2178, + "4": 0.17263, + "5": 0.17175, + "6": 0.1737, + "7": 0.30763, + "8": 0.17316, + "9": 0.17225, + "10": 0.17161, + "11": 0.17138, + "12": 0.17345, + "13": 0.1708, + "14": 0.17068, + "15": 0.17285, + "16": 0.17253, + "17": 0.17061, + "18": 0.17185, + "19": 0.17351, + "20": 0.175, + "21": 0.17435, + "22": 0.1757, + "23": 0.17509, + "24": 0.17747, + "25": 0.17587, + "26": 0.17538, + "27": 0.17486, + "28": 0.17624, + "29": 0.17684, + "30": 0.18129, + "31": 0.17765, + "32": 0.17748, + "33": 0.1765, + "34": 0.17555, + "35": 0.17641, + "36": 0.17791, + "37": 0.17413, + "38": 0.17296, + "39": 0.17138, + "40": 0.17121, + "41": 0.1743, + "42": 0.1743, + "43": 0.17289, + "44": 0.17309, + "45": 0.17515, + "46": 0.17365, + "47": 0.17469, + "48": 0.1739, + "49": 0.17459, + "50": 0.17254, + "51": 0.31772, + "52": 0.24147, + "53": 0.17617, + "54": 0.1725, + "55": 0.17331, + "56": 0.17461, + "57": 0.17487, + "58": 0.17259, + "59": 0.17142, + "60": 0.17331, + "61": 0.21904, + "62": 0.1749, + "63": 0.17348, + "64": 0.17228, + "65": 0.17548, + "66": 0.17296, + "67": 0.17392, + "68": 0.17424, + "69": 0.17422, + "70": 0.17328, + "71": 0.17267, + "72": 0.17553, + "73": 0.1743, + "74": 0.17571, + "75": 0.17546, + "76": 0.17697, + "77": 0.17466, + "78": 0.17461, + "79": 0.17611, + "80": 0.17747, + "81": 0.17367, + "82": 0.17383, + "83": 0.17486, + "84": 0.17354, + "85": 0.17187, + "86": 0.17195, + "87": 0.17399, + "88": 0.17131, + "89": 0.17336, + "90": 0.17405, + "91": 0.17509, + "92": 0.17582, + "93": 0.17555, + "94": 0.17561, + "95": 0.17688, + "96": 0.17761, + "97": 0.17433, + "98": 0.17661, + "99": 0.17639, + "100": 0.17545 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json index 194b1d83846..459e4d128b9 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, + "1": 10.95177, + "2": 10.94867, "3": 10.94863, "4": 10.94972, "5": 10.94752, - "6": 10.94473, - "7": 10.94653, - "8": 10.94474, - "9": 10.94189, - "10": 10.94477, - "11": 10.94104, - "12": 10.93297, - "13": 10.92309, - "14": 10.92594, - "15": 10.8997, - "16": 10.89084, - "17": 10.89814, - "18": 10.88338, - "19": 10.88983, - "20": 10.79733, - "21": 10.79296, - "22": 10.77998, - "23": 10.76962, - "24": 10.74573, - "25": 10.73534, - "26": 10.72509, - "27": 10.67795, - "28": 10.62002, - "29": 10.59163, - "30": 10.56328, - "31": 10.56511, - "32": 10.54391, - "33": 10.50913, + "6": 10.94481, + "7": 10.94654, + "8": 10.94475, + "9": 10.94192, + "10": 10.94475, + "11": 10.9411, + "12": 10.93294, + "13": 10.92301, + "14": 10.92591, + "15": 10.89965, + "16": 10.8908, + "17": 10.89826, + "18": 10.88342, + "19": 10.88985, + "20": 10.7974, + "21": 10.79298, + "22": 10.78007, + "23": 10.76967, + "24": 10.74584, + "25": 10.7353, + "26": 10.72505, + "27": 10.67792, + "28": 10.62003, + "29": 10.59162, + "30": 10.56332, + "31": 10.56518, + "32": 10.54399, + "33": 10.50912, "34": 10.48321, - "35": 10.46997, - "36": 10.44684, - "37": 10.42648, - "38": 10.43042, - "39": 10.39272, - "40": 10.38039, - "41": 10.35134, - "42": 10.32751, - "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24782, - "47": 10.2438, - "48": 10.19392, - "49": 10.1967, - "50": 10.18921, - "51": 10.19743, - "52": 10.15225, - "53": 10.15772, - "54": 10.12075, - "55": 10.09671, - "56": 10.12112, - "57": 10.1117, - "58": 10.12098, - "59": 10.0624, - "60": 10.09235, - "61": 10.04175, - "62": 10.00957, - "63": 10.07948, - "64": 10.03237, - "65": 10.00321, - "66": 10.03952, - "67": 10.0196, - "68": 9.9857, - "69": 10.00194, + "35": 10.46989, + "36": 10.44693, + "37": 10.42657, + "38": 10.43049, + "39": 10.39276, + "40": 10.38042, + "41": 10.35136, + "42": 10.32754, + "43": 10.31128, + "44": 10.28351, + "45": 10.29607, + "46": 10.24787, + "47": 10.24381, + "48": 10.19395, + "49": 10.19677, + "50": 10.18925, + "51": 10.19742, + "52": 10.15222, + "53": 10.15777, + "54": 10.12078, + "55": 10.09677, + "56": 10.12113, + "57": 10.11179, + "58": 10.12099, + "59": 10.06245, + "60": 10.09238, + "61": 10.04181, + "62": 10.0096, + "63": 10.07945, + "64": 10.03238, + "65": 10.00325, + "66": 10.03958, + "67": 10.01965, + "68": 9.98575, + "69": 10.00201, "70": 9.98145, - "71": 10.00455, - "72": 9.99623, - "73": 9.98762, - "74": 9.97285, - "75": 9.93231, + "71": 10.00458, + "72": 9.99624, + "73": 9.98765, + "74": 9.97292, + "75": 9.93235, "76": 9.96721, - "77": 9.96947, - "78": 9.92081, - "79": 9.91732, - "80": 9.93739, - "81": 9.95719, - "82": 9.89591, - "83": 9.86217, - "84": 9.80484, - "85": 9.78635, - "86": 9.89555, - "87": 9.90871, - "88": 9.88855, - "89": 9.83329, + "77": 9.96958, + "78": 9.92088, + "79": 9.91738, + "80": 9.93744, + "81": 9.95727, + "82": 9.89594, + "83": 9.86225, + "84": 9.80486, + "85": 9.78643, + "86": 9.89557, + "87": 9.90878, + "88": 9.88857, + "89": 9.83332, "90": 9.82425, "91": 9.83931, - "92": 9.82243, - "93": 9.75871, - "94": 9.83497, - "95": 9.82145, - "96": 9.81159, - "97": 9.75627, - "98": 9.78245, - "99": 9.82338, - "100": 9.71587 + "92": 9.82248, + "93": 9.75876, + "94": 9.83501, + "95": 9.82148, + "96": 9.8116, + "97": 9.75626, + "98": 9.78247, + "99": 9.82337, + "100": 9.71588 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 74.0, - "2": 75.0, - "3": 64.0, - "4": 85.0, - "5": 53.0, - "6": 73.0, - "7": 70.0, - "8": 69.0, - "9": 56.0, - "10": 54.0, - "11": 77.0, - "12": 82.0, - "13": 64.0, - "14": 71.0, - "15": 73.0, - "16": 68.0, - "17": 60.0, - "18": 85.0, - "19": 77.0, - "20": 66.0, - "21": 51.0, - "22": 65.0, - "23": 57.0, - "24": 75.0, - "25": 64.0, - "26": 75.0, - "27": 61.0, + "1": 52.0, + "2": 54.0, + "3": 67.0, + "4": 68.0, + "5": 50.0, + "6": 69.0, + "7": 69.0, + "8": 70.0, + "9": 68.0, + "10": 71.0, + "11": 78.0, + "12": 60.0, + "13": 79.0, + "14": 61.0, + "15": 71.0, + "16": 57.0, + "17": 52.0, + "18": 73.0, + "19": 57.0, + "20": 65.0, + "21": 82.0, + "22": 73.0, + "23": 73.0, + "24": 71.0, + "25": 51.0, + "26": 71.0, + "27": 57.0, "28": 64.0, - "29": 71.0, - "30": 58.0, - "31": 75.0, - "32": 58.0, - "33": 67.0, - "34": 74.0, - "35": 70.0, - "36": 85.0, - "37": 69.0, - "38": 77.0, - "39": 69.0, - "40": 87.0, - "41": 77.0, - "42": 79.0, - "43": 74.0, - "44": 95.0, - "45": 67.0, - "46": 63.0, - "47": 71.0, - "48": 76.0, - "49": 73.0, + "29": 70.0, + "30": 75.0, + "31": 89.0, + "32": 70.0, + "33": 69.0, + "34": 78.0, + "35": 79.0, + "36": 69.0, + "37": 83.0, + "38": 67.0, + "39": 71.0, + "40": 50.0, + "41": 79.0, + "42": 70.0, + "43": 83.0, + "44": 73.0, + "45": 87.0, + "46": 78.0, + "47": 90.0, + "48": 74.0, + "49": 84.0, "50": 79.0, - "51": 95.0, - "52": 84.0, - "53": 70.0, - "54": 91.0, - "55": 74.0, - "56": 78.0, - "57": 72.0, - "58": 78.0, - "59": 72.0, - "60": 86.0, - "61": 68.0, - "62": 98.0, - "63": 96.0, - "64": 93.0, - "65": 82.0, - "66": 66.0, - "67": 110.0, + "51": 102.0, + "52": 88.0, + "53": 89.0, + "54": 90.0, + "55": 75.0, + "56": 80.0, + "57": 60.0, + "58": 79.0, + "59": 66.0, + "60": 77.0, + "61": 83.0, + "62": 109.0, + "63": 72.0, + "64": 114.0, + "65": 86.0, + "66": 78.0, + "67": 102.0, "68": 96.0, - "69": 90.0, - "70": 101.0, - "71": 82.0, + "69": 79.0, + "70": 82.0, + "71": 68.0, "72": 76.0, - "73": 76.0, - "74": 87.0, - "75": 74.0, - "76": 74.0, - "77": 62.0, - "78": 77.0, - "79": 89.0, - "80": 68.0, - "81": 91.0, - "82": 73.0, - "83": 94.0, - "84": 90.0, - "85": 75.0, - "86": 94.0, - "87": 76.0, - "88": 80.0, - "89": 95.0, - "90": 96.0, - "91": 65.0, - "92": 65.0, - "93": 57.0, - "94": 78.0, - "95": 89.0, - "96": 60.0, - "97": 109.0, - "98": 56.0, - "99": 68.0, - "100": 99.0 + "73": 94.0, + "74": 75.0, + "75": 76.0, + "76": 82.0, + "77": 88.0, + "78": 119.0, + "79": 107.0, + "80": 82.0, + "81": 73.0, + "82": 72.0, + "83": 82.0, + "84": 80.0, + "85": 77.0, + "86": 77.0, + "87": 80.0, + "88": 77.0, + "89": 90.0, + "90": 92.0, + "91": 77.0, + "92": 72.0, + "93": 82.0, + "94": 72.0, + "95": 86.0, + "96": 83.0, + "97": 83.0, + "98": 74.0, + "99": 70.0, + "100": 72.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.05646, - "3": 1.64926, - "4": 1.51364, - "5": 0.87448, - "6": 0.8958, - "7": 1.13314, - "8": 1.69323, - "9": 1.97024, - "10": 0.889, - "11": 1.14997, - "12": 1.33119, - "13": 1.43251, - "14": 0.87966, - "15": 1.21883, - "16": 1.18965, - "17": 1.28078, - "18": 1.42888, - "19": 1.3082, - "20": 1.03096, - "21": 1.47204, - "22": 1.171, - "23": 1.21307, - "24": 1.75377, - "25": 0.91489, - "26": 1.11231, - "27": 1.4801, - "28": 1.77159, - "29": 0.80668, - "30": 1.10193, - "31": 0.97633, - "32": 1.61749, - "33": 1.55435, - "34": 1.15377, - "35": 1.71313, - "36": 0.91351, - "37": 1.14328, - "38": 1.37552, - "39": 1.5564, - "40": 2.68163, - "41": 1.41261, - "42": 1.38325, - "43": 1.2548, - "44": 2.14185, - "45": 1.32246, - "46": 1.27186, - "47": 1.57862, - "48": 0.87885, - "49": 1.5616, - "50": 1.51913, - "51": 0.20474, - "52": 1.49753, - "53": 1.41508, - "54": 1.50829, - "55": 1.53237, - "56": 1.13149, - "57": 1.31628, - "58": 1.41131, - "59": 1.15957, - "60": 1.23452, - "61": 1.61395, - "62": 1.09671, - "63": 1.39238, - "64": 1.398, - "65": 0.93338, - "66": 1.17371, - "67": 1.94759, - "68": 1.02385, - "69": 0.98696, - "70": 1.50737, - "71": 1.85133, - "72": 0.85571, - "73": 1.68757, - "74": 0.72292, - "75": 0.99087, - "76": 1.55542, - "77": 1.44187, - "78": 0.88968, - "79": 2.01173, - "80": 1.55881, - "81": 1.43639, - "82": 0.77707, - "83": 1.28231, - "84": 1.32696, - "85": 0.98777, - "86": 1.41843, - "87": 1.71853, - "88": 1.38841, - "89": 1.61498, - "90": 1.6885, - "91": 1.05804, - "92": 1.55219, - "93": 1.57285, - "94": 1.46847, - "95": 1.25271, - "96": 1.32851, - "97": 1.26383, - "98": 1.35329, - "99": 2.01532, - "100": 1.47822 + "2": 5.08187, + "3": 0.19131, + "4": 0.18838, + "5": 0.19016, + "6": 0.18779, + "7": 0.1893, + "8": 0.18768, + "9": 0.18849, + "10": 0.18692, + "11": 0.18615, + "12": 0.18928, + "13": 0.19201, + "14": 0.18849, + "15": 0.18733, + "16": 0.18638, + "17": 0.18749, + "18": 0.18968, + "19": 0.18923, + "20": 0.18729, + "21": 0.18767, + "22": 0.18685, + "23": 0.19283, + "24": 0.19143, + "25": 0.18995, + "26": 0.18665, + "27": 0.20157, + "28": 0.19779, + "29": 0.19999, + "30": 0.18833, + "31": 0.19471, + "32": 0.19677, + "33": 0.19774, + "34": 0.19442, + "35": 0.1892, + "36": 0.18864, + "37": 0.18889, + "38": 0.18855, + "39": 0.1932, + "40": 0.18852, + "41": 0.19335, + "42": 0.19035, + "43": 0.20144, + "44": 0.19284, + "45": 0.18817, + "46": 0.18826, + "47": 0.18896, + "48": 0.18886, + "49": 0.18824, + "50": 0.18919, + "51": 0.22534, + "52": 0.23234, + "53": 0.19298, + "54": 0.18877, + "55": 0.18783, + "56": 0.18922, + "57": 0.18677, + "58": 0.19271, + "59": 0.1908, + "60": 0.19353, + "61": 0.19305, + "62": 0.19623, + "63": 0.19386, + "64": 0.19456, + "65": 0.19315, + "66": 0.19423, + "67": 0.19107, + "68": 0.19427, + "69": 0.19305, + "70": 0.19072, + "71": 0.19039, + "72": 0.19057, + "73": 0.19533, + "74": 0.19459, + "75": 0.19542, + "76": 0.19373, + "77": 0.19368, + "78": 0.19599, + "79": 0.1933, + "80": 0.19228, + "81": 0.19366, + "82": 0.19168, + "83": 0.19583, + "84": 0.19971, + "85": 0.19335, + "86": 0.19174, + "87": 0.19266, + "88": 0.19361, + "89": 0.19254, + "90": 0.19143, + "91": 0.19258, + "92": 0.19117, + "93": 0.19271, + "94": 0.18737, + "95": 0.19089, + "96": 0.1897, + "97": 0.19218, + "98": 0.19016, + "99": 0.19064, + "100": 0.19257 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json index 06a2c6615e5..40522d10145 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92867, - "4": 10.92885, - "5": 10.93089, - "6": 10.92768, - "7": 10.92339, - "8": 10.92161, - "9": 10.92031, - "10": 10.91963, - "11": 10.91026, - "12": 10.91814, - "13": 10.91528, - "14": 10.89825, - "15": 10.88044, - "16": 10.86845, - "17": 10.86882, - "18": 10.86255, - "19": 10.86184, - "20": 10.77349, - "21": 10.75515, - "22": 10.75525, - "23": 10.7451, - "24": 10.71105, - "25": 10.71078, - "26": 10.69967, - "27": 10.65193, - "28": 10.5825, - "29": 10.55638, - "30": 10.54863, - "31": 10.52498, - "32": 10.51762, - "33": 10.48146, - "34": 10.45103, - "35": 10.44996, - "36": 10.43489, - "37": 10.39411, - "38": 10.39543, - "39": 10.36014, - "40": 10.35802, - "41": 10.32785, - "42": 10.307, - "43": 10.28209, - "44": 10.25605, - "45": 10.27043, - "46": 10.23405, - "47": 10.21883, - "48": 10.17023, - "49": 10.17586, - "50": 10.17856, - "51": 10.19226, - "52": 10.13722, - "53": 10.13551, - "54": 10.10522, - "55": 10.08051, - "56": 10.11117, - "57": 10.09597, - "58": 10.10933, - "59": 10.05625, - "60": 10.07399, - "61": 10.02772, - "62": 10.00029, - "63": 10.07319, - "64": 10.0369, - "65": 10.00343, + "2": 10.92832, + "3": 10.92865, + "4": 10.92887, + "5": 10.93092, + "6": 10.92771, + "7": 10.9234, + "8": 10.92162, + "9": 10.9203, + "10": 10.91967, + "11": 10.91022, + "12": 10.91827, + "13": 10.91529, + "14": 10.8983, + "15": 10.88051, + "16": 10.86839, + "17": 10.86885, + "18": 10.86249, + "19": 10.86187, + "20": 10.77337, + "21": 10.75517, + "22": 10.75534, + "23": 10.74513, + "24": 10.71113, + "25": 10.71087, + "26": 10.69964, + "27": 10.65196, + "28": 10.58251, + "29": 10.55639, + "30": 10.54867, + "31": 10.52503, + "32": 10.51767, + "33": 10.48145, + "34": 10.45107, + "35": 10.44997, + "36": 10.43491, + "37": 10.39413, + "38": 10.39554, + "39": 10.36016, + "40": 10.35805, + "41": 10.32787, + "42": 10.30705, + "43": 10.28218, + "44": 10.25608, + "45": 10.27046, + "46": 10.23411, + "47": 10.21893, + "48": 10.17027, + "49": 10.17593, + "50": 10.17859, + "51": 10.19236, + "52": 10.13724, + "53": 10.13555, + "54": 10.10525, + "55": 10.08055, + "56": 10.11124, + "57": 10.096, + "58": 10.10935, + "59": 10.05629, + "60": 10.07402, + "61": 10.02777, + "62": 10.00036, + "63": 10.07322, + "64": 10.03699, + "65": 10.00348, "66": 10.03036, "67": 10.00721, - "68": 9.97016, + "68": 9.97027, "69": 9.99154, - "70": 9.97454, - "71": 10.00064, - "72": 9.97777, - "73": 9.97168, - "74": 9.956, - "75": 9.93149, - "76": 9.96344, - "77": 9.95634, - "78": 9.90551, - "79": 9.91217, - "80": 9.92898, - "81": 9.95658, - "82": 9.89201, - "83": 9.85591, - "84": 9.78695, - "85": 9.78569, - "86": 9.88392, - "87": 9.91031, - "88": 9.88719, - "89": 9.81821, - "90": 9.81249, + "70": 9.97455, + "71": 10.00068, + "72": 9.97786, + "73": 9.97171, + "74": 9.95605, + "75": 9.93157, + "76": 9.96343, + "77": 9.9564, + "78": 9.90555, + "79": 9.91225, + "80": 9.92908, + "81": 9.95664, + "82": 9.892, + "83": 9.85597, + "84": 9.787, + "85": 9.78578, + "86": 9.88395, + "87": 9.91032, + "88": 9.88725, + "89": 9.81819, + "90": 9.8125, "91": 9.82781, - "92": 9.81661, - "93": 9.74996, - "94": 9.83091, - "95": 9.82648, - "96": 9.80705, - "97": 9.7439, - "98": 9.77871, - "99": 9.82058, - "100": 9.71401 + "92": 9.81663, + "93": 9.75004, + "94": 9.83095, + "95": 9.82651, + "96": 9.80706, + "97": 9.74396, + "98": 9.77875, + "99": 9.82057, + "100": 9.71407 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 76.0, - "2": 63.0, - "3": 65.0, - "4": 73.0, - "5": 70.0, - "6": 60.0, - "7": 65.0, - "8": 61.0, - "9": 60.0, - "10": 68.0, - "11": 69.0, - "12": 55.0, - "13": 72.0, - "14": 74.0, - "15": 52.0, - "16": 66.0, - "17": 56.0, - "18": 62.0, - "19": 57.0, + "1": 59.0, + "2": 45.0, + "3": 78.0, + "4": 62.0, + "5": 64.0, + "6": 58.0, + "7": 62.0, + "8": 54.0, + "9": 69.0, + "10": 67.0, + "11": 62.0, + "12": 70.0, + "13": 64.0, + "14": 50.0, + "15": 58.0, + "16": 61.0, + "17": 70.0, + "18": 52.0, + "19": 58.0, "20": 72.0, - "21": 60.0, - "22": 60.0, - "23": 57.0, - "24": 72.0, - "25": 71.0, - "26": 77.0, + "21": 57.0, + "22": 63.0, + "23": 70.0, + "24": 75.0, + "25": 59.0, + "26": 58.0, "27": 66.0, - "28": 68.0, - "29": 63.0, - "30": 68.0, - "31": 80.0, + "28": 55.0, + "29": 57.0, + "30": 85.0, + "31": 76.0, "32": 75.0, - "33": 60.0, - "34": 71.0, - "35": 74.0, - "36": 76.0, - "37": 70.0, - "38": 64.0, - "39": 76.0, - "40": 78.0, - "41": 72.0, - "42": 81.0, - "43": 73.0, - "44": 87.0, - "45": 94.0, - "46": 59.0, - "47": 76.0, - "48": 82.0, - "49": 75.0, - "50": 80.0, - "51": 74.0, - "52": 76.0, - "53": 76.0, - "54": 68.0, - "55": 58.0, - "56": 68.0, - "57": 73.0, - "58": 104.0, - "59": 94.0, - "60": 84.0, - "61": 85.0, - "62": 77.0, - "63": 71.0, - "64": 92.0, - "65": 78.0, - "66": 97.0, - "67": 89.0, - "68": 82.0, - "69": 98.0, - "70": 82.0, - "71": 96.0, - "72": 87.0, + "33": 85.0, + "34": 78.0, + "35": 81.0, + "36": 65.0, + "37": 64.0, + "38": 76.0, + "39": 83.0, + "40": 72.0, + "41": 88.0, + "42": 60.0, + "43": 82.0, + "44": 78.0, + "45": 69.0, + "46": 74.0, + "47": 72.0, + "48": 87.0, + "49": 83.0, + "50": 83.0, + "51": 87.0, + "52": 88.0, + "53": 88.0, + "54": 78.0, + "55": 90.0, + "56": 64.0, + "57": 79.0, + "58": 96.0, + "59": 89.0, + "60": 69.0, + "61": 96.0, + "62": 68.0, + "63": 79.0, + "64": 94.0, + "65": 86.0, + "66": 78.0, + "67": 86.0, + "68": 86.0, + "69": 76.0, + "70": 80.0, + "71": 83.0, + "72": 82.0, "73": 94.0, - "74": 80.0, - "75": 80.0, - "76": 69.0, - "77": 100.0, - "78": 88.0, - "79": 90.0, - "80": 70.0, - "81": 102.0, - "82": 60.0, - "83": 93.0, - "84": 85.0, - "85": 69.0, - "86": 81.0, - "87": 75.0, - "88": 80.0, - "89": 84.0, - "90": 81.0, - "91": 74.0, - "92": 62.0, - "93": 65.0, - "94": 81.0, - "95": 91.0, - "96": 83.0, - "97": 88.0, - "98": 66.0, - "99": 72.0, - "100": 91.0 + "74": 83.0, + "75": 70.0, + "76": 65.0, + "77": 90.0, + "78": 85.0, + "79": 94.0, + "80": 81.0, + "81": 98.0, + "82": 52.0, + "83": 98.0, + "84": 80.0, + "85": 86.0, + "86": 77.0, + "87": 73.0, + "88": 68.0, + "89": 88.0, + "90": 86.0, + "91": 82.0, + "92": 56.0, + "93": 80.0, + "94": 72.0, + "95": 83.0, + "96": 75.0, + "97": 84.0, + "98": 77.0, + "99": 98.0, + "100": 77.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.03088, - "3": 3.34876, - "4": 2.53582, - "5": 2.48058, - "6": 2.43762, - "7": 1.87523, - "8": 2.50536, - "9": 2.65386, - "10": 2.04284, - "11": 2.73294, - "12": 1.82067, - "13": 2.48704, - "14": 2.68492, - "15": 1.98056, - "16": 2.2973, - "17": 2.95035, - "18": 2.40728, - "19": 2.51638, - "20": 2.29895, - "21": 2.79292, - "22": 2.5803, - "23": 3.3125, - "24": 2.54299, - "25": 2.96965, - "26": 3.57266, - "27": 2.68069, - "28": 2.62391, - "29": 2.78523, - "30": 2.0347, - "31": 1.80005, - "32": 3.21385, - "33": 1.71827, - "34": 1.87217, - "35": 2.78285, - "36": 2.60662, - "37": 2.08884, - "38": 1.57836, - "39": 2.23851, - "40": 2.02535, - "41": 1.9462, - "42": 2.95662, - "43": 2.26434, - "44": 2.28424, - "45": 2.64872, - "46": 1.94126, - "47": 2.08261, - "48": 2.90839, - "49": 2.16688, - "50": 2.52428, - "51": 2.10448, - "52": 2.34448, - "53": 2.09852, - "54": 2.50284, - "55": 1.98433, - "56": 3.40748, - "57": 1.51938, - "58": 2.28296, - "59": 1.91628, - "60": 1.80878, - "61": 2.36241, - "62": 2.24451, - "63": 1.64096, - "64": 2.12785, - "65": 2.06667, - "66": 2.38939, - "67": 2.48401, - "68": 1.73357, - "69": 2.38218, - "70": 2.32129, - "71": 3.13742, - "72": 2.27983, - "73": 2.21754, - "74": 2.47275, - "75": 2.27557, - "76": 3.45149, - "77": 3.03361, - "78": 2.35039, - "79": 3.48119, - "80": 2.39514, - "81": 2.94682, - "82": 2.42092, - "83": 2.52566, - "84": 2.19555, - "85": 2.57184, - "86": 2.11493, - "87": 2.91445, - "88": 2.56316, - "89": 2.76157, - "90": 2.28355, - "91": 2.23103, - "92": 2.74864, - "93": 2.04853, - "94": 2.46703, - "95": 2.82977, - "96": 2.48175, - "97": 2.19588, - "98": 2.00735, - "99": 3.01988, - "100": 2.2927 + "2": 5.28963, + "3": 2.36129, + "4": 1.92294, + "5": 1.69428, + "6": 1.82378, + "7": 1.82252, + "8": 2.15841, + "9": 2.31136, + "10": 2.03275, + "11": 1.97542, + "12": 1.59488, + "13": 2.11685, + "14": 1.91789, + "15": 1.8849, + "16": 2.08582, + "17": 2.59059, + "18": 2.08472, + "19": 2.07718, + "20": 1.88486, + "21": 2.27661, + "22": 1.86028, + "23": 2.17562, + "24": 1.8131, + "25": 2.34476, + "26": 2.74742, + "27": 2.11547, + "28": 2.15049, + "29": 1.68109, + "30": 2.04559, + "31": 1.63351, + "32": 2.81524, + "33": 1.46831, + "34": 1.48591, + "35": 2.17344, + "36": 2.18007, + "37": 1.60354, + "38": 1.16552, + "39": 1.71459, + "40": 2.10236, + "41": 2.20354, + "42": 2.67624, + "43": 1.76211, + "44": 2.14987, + "45": 2.68941, + "46": 2.07364, + "47": 2.28609, + "48": 2.62899, + "49": 1.93357, + "50": 2.35874, + "51": 1.92525, + "52": 2.26098, + "53": 2.30384, + "54": 2.44516, + "55": 2.19308, + "56": 2.8336, + "57": 2.6439, + "58": 2.37799, + "59": 1.94238, + "60": 1.76656, + "61": 2.27672, + "62": 2.65955, + "63": 1.95363, + "64": 2.24358, + "65": 1.74144, + "66": 2.50214, + "67": 1.81496, + "68": 1.37698, + "69": 2.17996, + "70": 1.74227, + "71": 2.18329, + "72": 1.72369, + "73": 2.17385, + "74": 2.0125, + "75": 1.85346, + "76": 1.86938, + "77": 2.05565, + "78": 2.01998, + "79": 2.31866, + "80": 2.28252, + "81": 2.54773, + "82": 2.37287, + "83": 2.163, + "84": 2.12374, + "85": 1.81172, + "86": 2.01121, + "87": 2.78937, + "88": 2.30314, + "89": 2.37996, + "90": 2.18426, + "91": 2.58075, + "92": 2.90391, + "93": 2.04976, + "94": 2.12866, + "95": 2.34567, + "96": 2.51037, + "97": 2.17546, + "98": 2.2316, + "99": 2.82488, + "100": 2.17822 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json index 4a92d229bef..b85c0164613 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9565, - "2": 10.94849, + "1": 10.95648, + "2": 10.94841, "3": 10.95756, - "4": 10.94202, - "5": 10.95033, - "6": 10.94689, - "7": 10.95168, - "8": 10.93756, + "4": 10.94197, + "5": 10.95031, + "6": 10.94685, + "7": 10.95167, + "8": 10.93765, "9": 10.946, - "10": 10.93591, + "10": 10.93589, "11": 10.93621, - "12": 10.93466, - "13": 10.9283, - "14": 10.92195, - "15": 10.90624, - "16": 10.88784, - "17": 10.90025, - "18": 10.8854, - "19": 10.88839, + "12": 10.93465, + "13": 10.92835, + "14": 10.92198, + "15": 10.90617, + "16": 10.88788, + "17": 10.90026, + "18": 10.88531, + "19": 10.8884, "20": 10.80044, "21": 10.79131, - "22": 10.78415, - "23": 10.7737, - "24": 10.73683, - "25": 10.74957, + "22": 10.7842, + "23": 10.77375, + "24": 10.73693, + "25": 10.74959, "26": 10.7245, - "27": 10.68604, - "28": 10.60707, - "29": 10.58236, - "30": 10.57355, - "31": 10.5636, - "32": 10.54604, - "33": 10.51273, + "27": 10.68609, + "28": 10.60712, + "29": 10.58239, + "30": 10.57353, + "31": 10.56357, + "32": 10.54617, + "33": 10.51274, "34": 10.47148, - "35": 10.47211, - "36": 10.45977, - "37": 10.41727, - "38": 10.42779, - "39": 10.39098, - "40": 10.3733, - "41": 10.3571, - "42": 10.34119, - "43": 10.30726, - "44": 10.28288, - "45": 10.29155, - "46": 10.2606, - "47": 10.24268, - "48": 10.20194, - "49": 10.19165, - "50": 10.19699, - "51": 10.20243, - "52": 10.15296, - "53": 10.15364, - "54": 10.12006, - "55": 10.09602, - "56": 10.11913, - "57": 10.10459, - "58": 10.11673, - "59": 10.06722, - "60": 10.0821, - "61": 10.03525, - "62": 10.0084, - "63": 10.07559, - "64": 10.03887, - "65": 10.01135, - "66": 10.03512, - "67": 10.01125, - "68": 9.97552, - "69": 9.99771, - "70": 9.97808, - "71": 10.00604, - "72": 9.98123, - "73": 9.97729, - "74": 9.95828, - "75": 9.93282, - "76": 9.96575, - "77": 9.95901, - "78": 9.90785, - "79": 9.9156, - "80": 9.93586, - "81": 9.96096, - "82": 9.89425, - "83": 9.85733, - "84": 9.78597, - "85": 9.78541, - "86": 9.88264, - "87": 9.911, - "88": 9.88565, - "89": 9.81812, - "90": 9.80987, - "91": 9.82892, - "92": 9.81579, - "93": 9.74665, - "94": 9.83158, - "95": 9.82725, - "96": 9.80459, - "97": 9.74219, - "98": 9.77984, - "99": 9.82183, - "100": 9.71495 + "35": 10.47206, + "36": 10.45986, + "37": 10.41731, + "38": 10.42792, + "39": 10.39102, + "40": 10.37333, + "41": 10.35712, + "42": 10.34125, + "43": 10.30733, + "44": 10.28284, + "45": 10.29156, + "46": 10.26062, + "47": 10.24277, + "48": 10.20206, + "49": 10.19163, + "50": 10.19704, + "51": 10.20255, + "52": 10.153, + "53": 10.15373, + "54": 10.12016, + "55": 10.09607, + "56": 10.11911, + "57": 10.10468, + "58": 10.1168, + "59": 10.0673, + "60": 10.08207, + "61": 10.03526, + "62": 10.00851, + "63": 10.07564, + "64": 10.03897, + "65": 10.01134, + "66": 10.03521, + "67": 10.0113, + "68": 9.97557, + "69": 9.99777, + "70": 9.97813, + "71": 10.00608, + "72": 9.98127, + "73": 9.9773, + "74": 9.95832, + "75": 9.93286, + "76": 9.96582, + "77": 9.95905, + "78": 9.90789, + "79": 9.91562, + "80": 9.93592, + "81": 9.96107, + "82": 9.89429, + "83": 9.85732, + "84": 9.78596, + "85": 9.7855, + "86": 9.88266, + "87": 9.91101, + "88": 9.88572, + "89": 9.81818, + "90": 9.80994, + "91": 9.82894, + "92": 9.81578, + "93": 9.74667, + "94": 9.83165, + "95": 9.82728, + "96": 9.80466, + "97": 9.74224, + "98": 9.77992, + "99": 9.82189, + "100": 9.715 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1964.0, - "2": 1821.0, - "3": 1779.0, - "4": 1802.0, - "5": 1810.0, - "6": 1794.0, - "7": 1964.0, - "8": 1613.0, - "9": 1907.0, - "10": 1802.0, - "11": 1698.0, - "12": 1745.0, - "13": 1800.0, - "14": 1806.0, - "15": 1661.0, - "16": 1868.0, - "17": 1825.0, - "18": 1876.0, - "19": 1816.0, - "20": 1743.0, - "21": 1785.0, - "22": 1812.0, - "23": 1673.0, - "24": 1860.0, - "25": 1812.0, - "26": 1884.0, - "27": 1886.0, - "28": 1822.0, - "29": 2001.0, - "30": 1933.0, - "31": 2172.0, - "32": 2014.0, - "33": 2072.0, - "34": 2105.0, - "35": 2228.0, - "36": 2012.0, - "37": 2337.0, - "38": 2165.0, - "39": 2243.0, - "40": 2309.0, - "41": 2386.0, - "42": 2065.0, - "43": 2390.0, - "44": 2257.0, - "45": 2578.0, - "46": 2478.0, - "47": 2479.0, - "48": 2558.0, - "49": 2832.0, - "50": 2710.0, - "51": 2627.0, - "52": 2778.0, - "53": 2670.0, - "54": 2987.0, - "55": 2705.0, - "56": 2798.0, - "57": 2365.0, - "58": 3783.0, - "59": 3048.0, - "60": 3203.0, - "61": 2904.0, - "62": 3240.0, - "63": 3657.0, + "1": 1828.0, + "2": 1819.0, + "3": 1795.0, + "4": 1784.0, + "5": 1887.0, + "6": 1892.0, + "7": 1972.0, + "8": 1810.0, + "9": 1841.0, + "10": 1891.0, + "11": 1720.0, + "12": 1779.0, + "13": 1862.0, + "14": 1898.0, + "15": 1557.0, + "16": 1763.0, + "17": 1851.0, + "18": 1935.0, + "19": 1707.0, + "20": 1789.0, + "21": 1911.0, + "22": 1840.0, + "23": 1750.0, + "24": 1912.0, + "25": 1797.0, + "26": 1903.0, + "27": 1870.0, + "28": 1837.0, + "29": 2022.0, + "30": 1849.0, + "31": 2091.0, + "32": 2012.0, + "33": 2128.0, + "34": 2107.0, + "35": 2243.0, + "36": 2043.0, + "37": 2380.0, + "38": 2158.0, + "39": 2268.0, + "40": 2391.0, + "41": 2424.0, + "42": 1987.0, + "43": 2561.0, + "44": 2205.0, + "45": 2531.0, + "46": 2426.0, + "47": 2559.0, + "48": 2748.0, + "49": 2835.0, + "50": 2622.0, + "51": 2396.0, + "52": 2808.0, + "53": 2815.0, + "54": 2933.0, + "55": 2696.0, + "56": 2803.0, + "57": 2381.0, + "58": 3779.0, + "59": 3059.0, + "60": 3206.0, + "61": 2929.0, + "62": 3331.0, + "63": 3529.0, "64": 3799.0, - "65": 2840.0, - "66": 3341.0, - "67": 4109.0, - "68": 3451.0, - "69": 3223.0, - "70": 3382.0, - "71": 3255.0, - "72": 3002.0, - "73": 3454.0, - "74": 3346.0, - "75": 3175.0, - "76": 3356.0, - "77": 3887.0, - "78": 3531.0, - "79": 3387.0, - "80": 3337.0, - "81": 3679.0, - "82": 2864.0, - "83": 3235.0, - "84": 2979.0, - "85": 2827.0, - "86": 3134.0, - "87": 3058.0, - "88": 3162.0, - "89": 3164.0, - "90": 3878.0, - "91": 3099.0, - "92": 2925.0, - "93": 3097.0, - "94": 2982.0, - "95": 3223.0, - "96": 3271.0, - "97": 3554.0, - "98": 3407.0, - "99": 3347.0, - "100": 3429.0 + "65": 2809.0, + "66": 3340.0, + "67": 3934.0, + "68": 3480.0, + "69": 3209.0, + "70": 3454.0, + "71": 3245.0, + "72": 3082.0, + "73": 3552.0, + "74": 3349.0, + "75": 3284.0, + "76": 3382.0, + "77": 3896.0, + "78": 3431.0, + "79": 3447.0, + "80": 3313.0, + "81": 3778.0, + "82": 2908.0, + "83": 3179.0, + "84": 3094.0, + "85": 2775.0, + "86": 3057.0, + "87": 3073.0, + "88": 3055.0, + "89": 3236.0, + "90": 4075.0, + "91": 2920.0, + "92": 2932.0, + "93": 3089.0, + "94": 3040.0, + "95": 3267.0, + "96": 3325.0, + "97": 3596.0, + "98": 3477.0, + "99": 3306.0, + "100": 3443.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.83571, - "3": 0.25403, - "4": 0.24564, - "5": 0.2397, - "6": 0.24184, - "7": 0.24237, - "8": 0.23884, - "9": 0.2409, - "10": 0.2434, - "11": 0.24299, - "12": 0.24126, - "13": 0.24107, - "14": 0.24274, - "15": 0.24418, - "16": 0.24502, - "17": 0.24425, - "18": 0.24013, - "19": 0.24368, - "20": 0.2432, - "21": 0.24306, - "22": 0.24269, - "23": 0.24131, - "24": 0.24342, - "25": 0.24474, - "26": 0.24385, - "27": 0.24476, - "28": 0.24198, - "29": 0.24377, - "30": 0.24253, - "31": 0.24738, - "32": 0.24466, - "33": 0.24213, - "34": 0.24536, - "35": 0.24677, - "36": 0.24564, - "37": 0.24583, - "38": 0.24203, - "39": 0.24281, - "40": 0.24325, - "41": 0.24367, - "42": 0.24503, - "43": 0.2436, - "44": 0.24623, - "45": 0.24506, - "46": 0.24468, - "47": 0.24924, - "48": 0.24393, - "49": 0.24515, - "50": 0.24299, - "51": 0.33774, - "52": 0.30193, - "53": 0.24255, - "54": 0.2451, - "55": 0.24449, - "56": 0.24371, - "57": 0.24521, - "58": 0.24606, - "59": 0.24794, - "60": 0.2455, - "61": 0.24167, - "62": 0.24472, - "63": 0.24338, - "64": 0.2435, - "65": 0.24425, - "66": 0.24247, - "67": 0.24212, - "68": 0.24377, - "69": 0.24397, - "70": 0.24608, - "71": 0.24115, - "72": 0.24207, - "73": 0.2454, - "74": 0.24665, - "75": 0.24513, - "76": 0.24555, - "77": 0.2455, - "78": 0.24596, - "79": 0.24672, - "80": 0.24735, - "81": 0.24261, - "82": 0.24734, - "83": 0.24594, - "84": 0.24375, - "85": 0.24354, - "86": 0.24182, - "87": 0.24245, - "88": 0.2424, - "89": 0.24483, - "90": 0.24439, - "91": 0.24593, - "92": 0.24675, - "93": 0.24763, - "94": 0.24852, - "95": 0.24493, - "96": 0.24545, - "97": 0.24707, - "98": 0.25006, - "99": 0.24899, - "100": 0.24356 + "2": 4.29214, + "3": 0.24219, + "4": 0.20595, + "5": 0.20095, + "6": 0.20336, + "7": 0.20292, + "8": 0.20427, + "9": 0.20475, + "10": 0.20599, + "11": 0.20496, + "12": 0.20342, + "13": 0.20639, + "14": 0.20514, + "15": 0.2062, + "16": 0.20455, + "17": 0.20648, + "18": 0.20607, + "19": 0.20747, + "20": 0.20925, + "21": 0.20673, + "22": 0.21004, + "23": 0.20798, + "24": 0.20662, + "25": 0.2074, + "26": 0.20816, + "27": 0.20705, + "28": 0.2049, + "29": 0.20486, + "30": 0.20741, + "31": 0.20899, + "32": 0.20645, + "33": 0.21127, + "34": 0.20779, + "35": 0.20767, + "36": 0.20695, + "37": 0.20691, + "38": 0.20715, + "39": 0.20695, + "40": 0.21015, + "41": 0.20956, + "42": 0.20632, + "43": 0.21434, + "44": 0.20994, + "45": 0.20945, + "46": 0.20991, + "47": 0.20809, + "48": 0.20795, + "49": 0.20657, + "50": 0.20554, + "51": 0.33211, + "52": 0.26837, + "53": 0.21067, + "54": 0.20937, + "55": 0.20778, + "56": 0.20876, + "57": 0.20712, + "58": 0.20705, + "59": 0.20913, + "60": 0.20745, + "61": 0.20696, + "62": 0.2085, + "63": 0.2078, + "64": 0.20922, + "65": 0.20816, + "66": 0.21059, + "67": 0.21423, + "68": 0.20844, + "69": 0.20678, + "70": 0.20828, + "71": 0.20515, + "72": 0.20616, + "73": 0.20726, + "74": 0.20708, + "75": 0.20711, + "76": 0.20661, + "77": 0.21039, + "78": 0.21017, + "79": 0.20777, + "80": 0.20774, + "81": 0.20731, + "82": 0.20752, + "83": 0.20849, + "84": 0.21089, + "85": 0.20996, + "86": 0.21006, + "87": 0.20837, + "88": 0.21502, + "89": 0.21357, + "90": 0.2108, + "91": 0.20793, + "92": 0.20798, + "93": 0.21157, + "94": 0.20816, + "95": 0.20936, + "96": 0.20891, + "97": 0.21407, + "98": 0.2102, + "99": 0.21062, + "100": 0.20734 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json index b111a92e440..41b32623e82 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9313, - "2": 10.92649, - "3": 10.92492, - "4": 10.92049, - "5": 10.91862, - "6": 10.91576, - "7": 10.9247, - "8": 10.91818, - "9": 10.9151, - "10": 10.91994, - "11": 10.91158, - "12": 10.90479, - "13": 10.90054, - "14": 10.9013, - "15": 10.88359, + "1": 10.93124, + "2": 10.92656, + "3": 10.92483, + "4": 10.92047, + "5": 10.9186, + "6": 10.91565, + "7": 10.92468, + "8": 10.91821, + "9": 10.91516, + "10": 10.91989, + "11": 10.91162, + "12": 10.90475, + "13": 10.90051, + "14": 10.90133, + "15": 10.88358, "16": 10.8637, - "17": 10.88307, - "18": 10.86572, - "19": 10.86865, - "20": 10.79164, - "21": 10.7875, - "22": 10.7756, - "23": 10.76381, - "24": 10.73309, - "25": 10.73329, - "26": 10.71734, - "27": 10.6737, - "28": 10.61526, - "29": 10.58911, - "30": 10.5584, - "31": 10.56339, + "17": 10.88315, + "18": 10.86578, + "19": 10.86864, + "20": 10.79161, + "21": 10.78749, + "22": 10.77557, + "23": 10.76385, + "24": 10.73307, + "25": 10.73334, + "26": 10.71736, + "27": 10.67378, + "28": 10.61523, + "29": 10.5891, + "30": 10.55845, + "31": 10.56337, "32": 10.54733, - "33": 10.51065, - "34": 10.48411, - "35": 10.48015, - "36": 10.45812, - "37": 10.43332, - "38": 10.43472, - "39": 10.39235, - "40": 10.38663, - "41": 10.36414, + "33": 10.51066, + "34": 10.48413, + "35": 10.48022, + "36": 10.45815, + "37": 10.43336, + "38": 10.43487, + "39": 10.39242, + "40": 10.38667, + "41": 10.36427, "42": 10.34637, - "43": 10.32167, - "44": 10.29975, - "45": 10.30678, - "46": 10.27001, - "47": 10.25529, - "48": 10.20947, - "49": 10.21406, - "50": 10.20718, - "51": 10.20963, - "52": 10.16215, + "43": 10.32175, + "44": 10.2997, + "45": 10.30682, + "46": 10.27007, + "47": 10.25537, + "48": 10.20952, + "49": 10.21412, + "50": 10.20724, + "51": 10.20967, + "52": 10.1622, "53": 10.16867, - "54": 10.13116, - "55": 10.10553, - "56": 10.13037, - "57": 10.1174, - "58": 10.12717, - "59": 10.07187, - "60": 10.09529, + "54": 10.13123, + "55": 10.10561, + "56": 10.1304, + "57": 10.11749, + "58": 10.12723, + "59": 10.07194, + "60": 10.09526, "61": 10.04934, - "62": 10.01408, - "63": 10.08517, - "64": 10.0346, - "65": 10.00773, - "66": 10.0456, - "67": 10.02072, - "68": 9.98561, - "69": 9.99975, - "70": 9.98442, + "62": 10.01411, + "63": 10.08524, + "64": 10.03467, + "65": 10.00779, + "66": 10.04566, + "67": 10.02073, + "68": 9.98565, + "69": 9.99973, + "70": 9.98439, "71": 10.00594, - "72": 9.99176, - "73": 9.98241, - "74": 9.9695, - "75": 9.92723, + "72": 9.9918, + "73": 9.98247, + "74": 9.96949, + "75": 9.92731, "76": 9.96628, - "77": 9.97061, - "78": 9.91891, - "79": 9.91537, - "80": 9.93334, - "81": 9.95468, - "82": 9.89573, - "83": 9.86367, - "84": 9.80477, - "85": 9.78775, + "77": 9.97068, + "78": 9.91894, + "79": 9.91539, + "80": 9.93338, + "81": 9.95475, + "82": 9.89574, + "83": 9.8637, + "84": 9.80473, + "85": 9.7878, "86": 9.89713, - "87": 9.90766, - "88": 9.88965, - "89": 9.83169, + "87": 9.90773, + "88": 9.88967, + "89": 9.83167, "90": 9.82537, - "91": 9.83475, - "92": 9.8245, - "93": 9.75847, - "94": 9.83101, - "95": 9.81948, - "96": 9.80777, - "97": 9.75345, - "98": 9.77888, - "99": 9.82077, - "100": 9.71277 + "91": 9.83482, + "92": 9.82443, + "93": 9.75854, + "94": 9.83105, + "95": 9.81946, + "96": 9.80783, + "97": 9.75353, + "98": 9.77882, + "99": 9.82076, + "100": 9.71284 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1952.0, - "2": 1817.0, - "3": 1808.0, - "4": 1791.0, - "5": 1885.0, + "1": 1979.0, + "2": 1748.0, + "3": 1815.0, + "4": 1776.0, + "5": 1902.0, "6": 1821.0, - "7": 2063.0, - "8": 1849.0, - "9": 1891.0, - "10": 1826.0, - "11": 1829.0, - "12": 1803.0, - "13": 1971.0, - "14": 1928.0, - "15": 1702.0, - "16": 1851.0, - "17": 1860.0, - "18": 1860.0, - "19": 1847.0, - "20": 1759.0, - "21": 2068.0, - "22": 1814.0, - "23": 1824.0, - "24": 1928.0, - "25": 1823.0, - "26": 1823.0, - "27": 1934.0, - "28": 2025.0, - "29": 2099.0, - "30": 1913.0, - "31": 2038.0, - "32": 2141.0, - "33": 2100.0, - "34": 2173.0, - "35": 2211.0, - "36": 2318.0, - "37": 2437.0, - "38": 2196.0, - "39": 2330.0, - "40": 2327.0, - "41": 2312.0, - "42": 2141.0, - "43": 2494.0, - "44": 2427.0, - "45": 2658.0, - "46": 2486.0, - "47": 2586.0, - "48": 2662.0, - "49": 2805.0, - "50": 2596.0, - "51": 2668.0, - "52": 2908.0, - "53": 2663.0, - "54": 3035.0, - "55": 2510.0, - "56": 2886.0, - "57": 2353.0, - "58": 3662.0, - "59": 2903.0, - "60": 3034.0, - "61": 2731.0, - "62": 3185.0, - "63": 3430.0, - "64": 3798.0, - "65": 2668.0, - "66": 2992.0, - "67": 3917.0, - "68": 3432.0, - "69": 3100.0, - "70": 3370.0, - "71": 3312.0, - "72": 3110.0, - "73": 3537.0, - "74": 3315.0, - "75": 3231.0, - "76": 3512.0, - "77": 3745.0, - "78": 3437.0, - "79": 3539.0, - "80": 3091.0, - "81": 3446.0, - "82": 3428.0, - "83": 3132.0, - "84": 3071.0, - "85": 2856.0, - "86": 3102.0, - "87": 3132.0, - "88": 3289.0, - "89": 3119.0, - "90": 4321.0, - "91": 3021.0, - "92": 3215.0, - "93": 3117.0, - "94": 3434.0, - "95": 3353.0, - "96": 3968.0, - "97": 3613.0, - "98": 3539.0, - "99": 3548.0, - "100": 3423.0 + "7": 2096.0, + "8": 1814.0, + "9": 1986.0, + "10": 1854.0, + "11": 1841.0, + "12": 1830.0, + "13": 1959.0, + "14": 1959.0, + "15": 1732.0, + "16": 1757.0, + "17": 1857.0, + "18": 1843.0, + "19": 1871.0, + "20": 1811.0, + "21": 1908.0, + "22": 1769.0, + "23": 1916.0, + "24": 1982.0, + "25": 1724.0, + "26": 1815.0, + "27": 1991.0, + "28": 2083.0, + "29": 2080.0, + "30": 1957.0, + "31": 2135.0, + "32": 2036.0, + "33": 2046.0, + "34": 2092.0, + "35": 2270.0, + "36": 2223.0, + "37": 2385.0, + "38": 2130.0, + "39": 2311.0, + "40": 2405.0, + "41": 2399.0, + "42": 2119.0, + "43": 2534.0, + "44": 2262.0, + "45": 2600.0, + "46": 2520.0, + "47": 2549.0, + "48": 2614.0, + "49": 2830.0, + "50": 2624.0, + "51": 2573.0, + "52": 2725.0, + "53": 2640.0, + "54": 2861.0, + "55": 2646.0, + "56": 2801.0, + "57": 2420.0, + "58": 3678.0, + "59": 2935.0, + "60": 3002.0, + "61": 2701.0, + "62": 3221.0, + "63": 3238.0, + "64": 3778.0, + "65": 2746.0, + "66": 3130.0, + "67": 3794.0, + "68": 3423.0, + "69": 3104.0, + "70": 3405.0, + "71": 3326.0, + "72": 3135.0, + "73": 3547.0, + "74": 3247.0, + "75": 3221.0, + "76": 3389.0, + "77": 3780.0, + "78": 3334.0, + "79": 3528.0, + "80": 3092.0, + "81": 3390.0, + "82": 3497.0, + "83": 3199.0, + "84": 3098.0, + "85": 2849.0, + "86": 3093.0, + "87": 3255.0, + "88": 3214.0, + "89": 3222.0, + "90": 4190.0, + "91": 2958.0, + "92": 3147.0, + "93": 3180.0, + "94": 3453.0, + "95": 3333.0, + "96": 3907.0, + "97": 3640.0, + "98": 3579.0, + "99": 3534.0, + "100": 3385.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.98595, - "3": 0.22249, - "4": 0.21742, - "5": 0.22431, - "6": 0.21965, - "7": 0.22012, - "8": 0.21964, - "9": 0.21811, - "10": 0.22202, - "11": 0.21921, - "12": 0.21886, - "13": 0.21819, - "14": 0.21831, - "15": 0.22111, - "16": 0.22122, - "17": 0.22218, - "18": 0.22075, - "19": 0.22666, - "20": 0.2194, - "21": 0.21989, - "22": 0.21964, - "23": 0.21747, - "24": 0.21849, - "25": 0.21815, - "26": 0.21631, - "27": 0.21691, - "28": 0.21882, - "29": 0.22592, - "30": 0.22007, - "31": 0.22101, - "32": 0.21883, - "33": 0.22271, - "34": 0.2218, - "35": 0.22044, - "36": 0.21738, - "37": 0.21825, - "38": 0.21973, - "39": 0.21709, - "40": 0.21702, - "41": 0.21984, - "42": 0.22062, - "43": 0.2206, - "44": 0.21837, - "45": 0.21769, - "46": 0.21981, - "47": 0.21851, - "48": 0.21744, - "49": 0.2182, - "50": 0.21878, - "51": 0.25555, - "52": 0.26714, - "53": 0.22308, - "54": 0.22545, - "55": 0.22025, - "56": 0.22196, - "57": 0.22446, - "58": 0.21756, - "59": 0.21875, - "60": 0.21983, - "61": 0.22253, - "62": 0.2224, - "63": 0.2195, - "64": 0.221, - "65": 0.21958, - "66": 0.22081, - "67": 0.21709, - "68": 0.21941, - "69": 0.23791, - "70": 0.21957, - "71": 0.22017, - "72": 0.21767, - "73": 0.21989, - "74": 0.21807, - "75": 0.22008, - "76": 0.2181, - "77": 0.22237, - "78": 0.21893, - "79": 0.22044, - "80": 0.21745, - "81": 0.21784, - "82": 0.21951, - "83": 0.21827, - "84": 0.21882, - "85": 0.21898, - "86": 0.21833, - "87": 0.21788, - "88": 0.2192, - "89": 0.219, - "90": 0.21738, - "91": 0.21787, - "92": 0.21744, - "93": 0.2176, - "94": 0.21938, - "95": 0.22201, - "96": 0.21719, - "97": 0.21827, - "98": 0.2177, - "99": 0.21711, - "100": 0.21697 + "2": 3.14897, + "3": 0.23967, + "4": 0.231, + "5": 0.23194, + "6": 0.23013, + "7": 0.22883, + "8": 0.23541, + "9": 0.22739, + "10": 0.23588, + "11": 0.22875, + "12": 0.23239, + "13": 0.22883, + "14": 0.23266, + "15": 0.22884, + "16": 0.23118, + "17": 0.22919, + "18": 0.23631, + "19": 0.25228, + "20": 0.22998, + "21": 0.23335, + "22": 0.23118, + "23": 0.23583, + "24": 0.22703, + "25": 0.22781, + "26": 0.22928, + "27": 0.23815, + "28": 0.22793, + "29": 0.22905, + "30": 0.22953, + "31": 0.22839, + "32": 0.22771, + "33": 0.23594, + "34": 0.23224, + "35": 0.22928, + "36": 0.2262, + "37": 0.22954, + "38": 0.22528, + "39": 0.22587, + "40": 0.22772, + "41": 0.22665, + "42": 0.22847, + "43": 0.22794, + "44": 0.22734, + "45": 0.22798, + "46": 0.23032, + "47": 0.22459, + "48": 0.22993, + "49": 0.22623, + "50": 0.2267, + "51": 0.26212, + "52": 0.27619, + "53": 0.23168, + "54": 0.228, + "55": 0.22817, + "56": 0.22987, + "57": 0.23074, + "58": 0.22926, + "59": 0.22857, + "60": 0.2265, + "61": 0.22711, + "62": 0.22711, + "63": 0.22478, + "64": 0.22583, + "65": 0.2287, + "66": 0.22771, + "67": 0.22543, + "68": 0.2298, + "69": 0.22446, + "70": 0.22549, + "71": 0.22675, + "72": 0.22709, + "73": 0.22624, + "74": 0.22552, + "75": 0.22498, + "76": 0.22399, + "77": 0.22623, + "78": 0.22806, + "79": 0.22657, + "80": 0.22514, + "81": 0.22883, + "82": 0.22737, + "83": 0.22732, + "84": 0.22572, + "85": 0.22544, + "86": 0.2252, + "87": 0.22494, + "88": 0.22641, + "89": 0.22719, + "90": 0.22829, + "91": 0.22568, + "92": 0.22493, + "93": 0.22547, + "94": 0.22645, + "95": 0.22657, + "96": 0.23791, + "97": 0.2309, + "98": 0.2308, + "99": 0.22697, + "100": 0.22711 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json index fa22b14865c..12ebfdd8ef1 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95307, - "2": 10.94856, - "3": 10.95609, - "4": 10.94434, - "5": 10.94637, - "6": 10.94557, - "7": 10.95292, - "8": 10.93872, - "9": 10.94966, - "10": 10.93736, - "11": 10.93794, - "12": 10.93034, - "13": 10.92636, - "14": 10.92328, - "15": 10.90361, - "16": 10.89261, - "17": 10.90125, + "1": 10.95306, + "2": 10.94859, + "3": 10.95615, + "4": 10.94432, + "5": 10.94639, + "6": 10.94553, + "7": 10.95287, + "8": 10.93869, + "9": 10.94969, + "10": 10.93737, + "11": 10.93802, + "12": 10.93045, + "13": 10.92628, + "14": 10.92329, + "15": 10.90359, + "16": 10.89263, + "17": 10.90118, "18": 10.88605, - "19": 10.88664, - "20": 10.80354, - "21": 10.79325, - "22": 10.78345, - "23": 10.77589, - "24": 10.73859, - "25": 10.75397, - "26": 10.72401, - "27": 10.68087, - "28": 10.60753, - "29": 10.58173, + "19": 10.88667, + "20": 10.80348, + "21": 10.79333, + "22": 10.78352, + "23": 10.77607, + "24": 10.73853, + "25": 10.75406, + "26": 10.72412, + "27": 10.68092, + "28": 10.60761, + "29": 10.58179, "30": 10.57586, - "31": 10.55865, - "32": 10.54534, - "33": 10.51316, - "34": 10.47157, - "35": 10.47226, - "36": 10.46033, - "37": 10.41614, - "38": 10.42803, - "39": 10.38981, - "40": 10.37324, + "31": 10.5587, + "32": 10.54539, + "33": 10.51327, + "34": 10.47162, + "35": 10.47231, + "36": 10.46037, + "37": 10.41615, + "38": 10.42801, + "39": 10.38986, + "40": 10.3732, "41": 10.35698, - "42": 10.34258, - "43": 10.30625, - "44": 10.2833, - "45": 10.29373, - "46": 10.26051, - "47": 10.24268, - "48": 10.2011, - "49": 10.1927, - "50": 10.19551, - "51": 10.20263, - "52": 10.15249, + "42": 10.34265, + "43": 10.30637, + "44": 10.28335, + "45": 10.2938, + "46": 10.26052, + "47": 10.24273, + "48": 10.20119, + "49": 10.19276, + "50": 10.19553, + "51": 10.20267, + "52": 10.15253, "53": 10.15361, - "54": 10.11924, - "55": 10.09675, - "56": 10.11854, - "57": 10.10417, - "58": 10.11747, - "59": 10.06702, + "54": 10.11928, + "55": 10.09682, + "56": 10.11859, + "57": 10.10424, + "58": 10.11749, + "59": 10.06707, "60": 10.08112, - "61": 10.03586, - "62": 10.00888, - "63": 10.07618, - "64": 10.03927, - "65": 10.01139, - "66": 10.03452, - "67": 10.01084, - "68": 9.97493, - "69": 9.99636, - "70": 9.97873, - "71": 10.00559, - "72": 9.98062, + "61": 10.03592, + "62": 10.00896, + "63": 10.07624, + "64": 10.03928, + "65": 10.01145, + "66": 10.03455, + "67": 10.0109, + "68": 9.97495, + "69": 9.99643, + "70": 9.97875, + "71": 10.00566, + "72": 9.98064, "73": 9.97765, - "74": 9.95924, - "75": 9.9335, - "76": 9.96535, + "74": 9.95934, + "75": 9.93357, + "76": 9.96538, "77": 9.9598, - "78": 9.9074, - "79": 9.91725, - "80": 9.93605, - "81": 9.96051, - "82": 9.89289, - "83": 9.85645, - "84": 9.78625, - "85": 9.78401, - "86": 9.88327, - "87": 9.90989, - "88": 9.88512, - "89": 9.81694, - "90": 9.80996, - "91": 9.82887, - "92": 9.81612, - "93": 9.74684, - "94": 9.83238, - "95": 9.8273, - "96": 9.80575, - "97": 9.74064, - "98": 9.77907, - "99": 9.82087, - "100": 9.71283 + "78": 9.90744, + "79": 9.91735, + "80": 9.93615, + "81": 9.9606, + "82": 9.89295, + "83": 9.85654, + "84": 9.78626, + "85": 9.78407, + "86": 9.88333, + "87": 9.90984, + "88": 9.88523, + "89": 9.81703, + "90": 9.80999, + "91": 9.82893, + "92": 9.81614, + "93": 9.7469, + "94": 9.83241, + "95": 9.82735, + "96": 9.80576, + "97": 9.74068, + "98": 9.77912, + "99": 9.82089, + "100": 9.7129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1879.0, - "2": 1840.0, - "3": 1763.0, - "4": 1799.0, - "5": 1826.0, - "6": 1799.0, - "7": 1948.0, - "8": 1699.0, - "9": 1844.0, - "10": 1821.0, - "11": 1760.0, - "12": 1737.0, - "13": 1900.0, - "14": 1871.0, - "15": 1586.0, - "16": 1801.0, - "17": 1831.0, - "18": 1844.0, - "19": 1719.0, - "20": 1793.0, - "21": 1923.0, - "22": 1777.0, - "23": 1785.0, - "24": 1835.0, - "25": 1854.0, - "26": 1897.0, - "27": 1952.0, - "28": 1885.0, - "29": 2125.0, - "30": 1983.0, - "31": 2149.0, - "32": 2115.0, - "33": 2121.0, - "34": 2212.0, - "35": 2218.0, - "36": 2118.0, - "37": 2349.0, - "38": 2207.0, - "39": 2256.0, - "40": 2381.0, - "41": 2353.0, - "42": 2050.0, - "43": 2518.0, - "44": 2259.0, - "45": 2659.0, - "46": 2517.0, - "47": 2552.0, - "48": 2676.0, - "49": 2840.0, - "50": 2551.0, - "51": 2495.0, - "52": 2765.0, - "53": 2808.0, - "54": 2907.0, - "55": 2675.0, - "56": 2865.0, - "57": 2242.0, - "58": 3810.0, - "59": 3052.0, - "60": 3190.0, + "1": 1873.0, + "2": 1839.0, + "3": 1794.0, + "4": 1707.0, + "5": 1848.0, + "6": 1769.0, + "7": 1956.0, + "8": 1758.0, + "9": 1923.0, + "10": 1783.0, + "11": 1753.0, + "12": 1736.0, + "13": 1729.0, + "14": 1907.0, + "15": 1662.0, + "16": 1764.0, + "17": 1859.0, + "18": 1837.0, + "19": 1818.0, + "20": 1795.0, + "21": 1814.0, + "22": 1785.0, + "23": 1794.0, + "24": 1819.0, + "25": 1851.0, + "26": 1917.0, + "27": 1897.0, + "28": 1915.0, + "29": 2081.0, + "30": 1907.0, + "31": 2059.0, + "32": 2110.0, + "33": 1994.0, + "34": 2153.0, + "35": 2197.0, + "36": 2055.0, + "37": 2273.0, + "38": 2054.0, + "39": 2186.0, + "40": 2307.0, + "41": 2398.0, + "42": 2097.0, + "43": 2504.0, + "44": 2206.0, + "45": 2602.0, + "46": 2509.0, + "47": 2512.0, + "48": 2738.0, + "49": 2728.0, + "50": 2606.0, + "51": 2591.0, + "52": 2784.0, + "53": 2689.0, + "54": 2785.0, + "55": 2711.0, + "56": 2875.0, + "57": 2338.0, + "58": 3839.0, + "59": 2922.0, + "60": 3073.0, "61": 2925.0, - "62": 3385.0, - "63": 3602.0, - "64": 3723.0, - "65": 2785.0, - "66": 3298.0, - "67": 4072.0, - "68": 3407.0, - "69": 3096.0, - "70": 3528.0, - "71": 3225.0, - "72": 3092.0, - "73": 3475.0, - "74": 3362.0, - "75": 3195.0, - "76": 3320.0, - "77": 3944.0, - "78": 3522.0, - "79": 3281.0, - "80": 3241.0, - "81": 3763.0, - "82": 2964.0, - "83": 3107.0, - "84": 3076.0, - "85": 2865.0, - "86": 2990.0, - "87": 3080.0, - "88": 3170.0, - "89": 3342.0, - "90": 3855.0, - "91": 2997.0, - "92": 2860.0, - "93": 3091.0, - "94": 3006.0, - "95": 3393.0, - "96": 3323.0, - "97": 3534.0, - "98": 3263.0, - "99": 3271.0, - "100": 3436.0 + "62": 3225.0, + "63": 3690.0, + "64": 3678.0, + "65": 2853.0, + "66": 3372.0, + "67": 4031.0, + "68": 3470.0, + "69": 3153.0, + "70": 3391.0, + "71": 3349.0, + "72": 2993.0, + "73": 3441.0, + "74": 3384.0, + "75": 3209.0, + "76": 3335.0, + "77": 3858.0, + "78": 3425.0, + "79": 3372.0, + "80": 3288.0, + "81": 3781.0, + "82": 2999.0, + "83": 3213.0, + "84": 2975.0, + "85": 2882.0, + "86": 3037.0, + "87": 3065.0, + "88": 3094.0, + "89": 3301.0, + "90": 3877.0, + "91": 3014.0, + "92": 2822.0, + "93": 3114.0, + "94": 2927.0, + "95": 3310.0, + "96": 3391.0, + "97": 3558.0, + "98": 3160.0, + "99": 3336.0, + "100": 3471.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.94756, - "3": 1.33968, - "4": 1.18099, - "5": 0.79196, - "6": 1.3329, - "7": 1.19878, - "8": 1.44642, - "9": 1.63098, - "10": 0.96334, - "11": 1.18733, - "12": 1.17451, - "13": 1.25407, - "14": 1.5443, - "15": 0.95591, - "16": 1.03101, - "17": 1.42579, - "18": 1.22868, - "19": 1.20668, - "20": 1.13172, - "21": 1.09404, - "22": 1.07889, - "23": 1.26565, - "24": 1.60129, - "25": 0.77105, - "26": 1.04269, - "27": 1.12442, - "28": 1.10423, - "29": 1.00549, - "30": 1.00908, - "31": 0.83743, - "32": 1.90219, - "33": 1.16577, - "34": 0.4742, - "35": 1.40055, - "36": 1.52214, - "37": 1.04511, - "38": 0.67596, - "39": 1.26218, - "40": 1.49779, - "41": 1.12567, - "42": 1.20837, - "43": 1.08314, - "44": 1.40335, - "45": 0.96504, - "46": 0.81372, - "47": 1.18445, - "48": 1.15653, - "49": 1.01329, - "50": 1.35972, - "51": 0.49238, - "52": 0.99614, - "53": 1.27125, - "54": 1.4193, - "55": 1.18832, - "56": 1.0429, - "57": 0.84399, - "58": 1.20729, - "59": 0.96425, - "60": 0.80519, - "61": 1.1901, - "62": 1.22612, - "63": 1.07342, - "64": 1.3296, - "65": 0.77658, - "66": 1.01715, - "67": 1.07719, - "68": 0.88113, - "69": 1.38992, - "70": 0.98348, - "71": 1.11555, - "72": 1.19298, - "73": 0.9969, - "74": 0.67233, - "75": 1.09868, - "76": 1.19409, - "77": 1.30202, - "78": 1.42778, - "79": 1.87638, - "80": 1.60954, - "81": 1.1351, - "82": 1.46354, - "83": 1.31014, - "84": 1.10204, - "85": 0.76669, - "86": 1.15568, - "87": 1.87379, - "88": 1.14966, - "89": 1.9549, - "90": 1.44848, - "91": 1.08126, - "92": 1.55667, - "93": 1.34264, - "94": 1.70687, - "95": 1.03485, - "96": 1.9428, - "97": 1.07111, - "98": 0.52931, - "99": 1.91464, - "100": 1.25041 + "2": 3.58576, + "3": 1.33379, + "4": 1.00859, + "5": 0.89556, + "6": 1.09149, + "7": 1.0284, + "8": 1.39362, + "9": 1.38455, + "10": 0.89798, + "11": 1.0201, + "12": 1.01825, + "13": 1.08874, + "14": 1.33281, + "15": 0.87526, + "16": 0.91185, + "17": 1.21116, + "18": 1.00936, + "19": 1.12078, + "20": 1.10838, + "21": 0.94639, + "22": 0.96563, + "23": 1.12519, + "24": 1.56235, + "25": 0.66814, + "26": 0.95846, + "27": 0.95081, + "28": 1.18238, + "29": 0.81908, + "30": 0.90515, + "31": 0.79417, + "32": 1.73286, + "33": 1.0358, + "34": 0.38887, + "35": 1.27277, + "36": 1.28199, + "37": 0.97944, + "38": 0.61739, + "39": 1.13075, + "40": 1.37583, + "41": 0.97022, + "42": 1.27869, + "43": 1.06224, + "44": 1.30505, + "45": 0.60249, + "46": 0.99157, + "47": 1.11627, + "48": 1.16393, + "49": 0.93448, + "50": 1.21412, + "51": 0.46154, + "52": 0.96049, + "53": 1.19112, + "54": 1.25769, + "55": 1.1169, + "56": 0.94462, + "57": 0.81632, + "58": 1.11741, + "59": 0.93143, + "60": 0.65542, + "61": 1.06897, + "62": 1.21356, + "63": 0.93403, + "64": 1.21153, + "65": 0.78809, + "66": 1.07764, + "67": 1.18749, + "68": 0.95992, + "69": 1.33825, + "70": 1.00512, + "71": 1.03861, + "72": 1.02794, + "73": 0.99144, + "74": 0.71221, + "75": 0.92341, + "76": 1.13042, + "77": 1.23259, + "78": 1.22797, + "79": 1.557, + "80": 1.00356, + "81": 1.05008, + "82": 1.23243, + "83": 1.18783, + "84": 1.02141, + "85": 0.66548, + "86": 0.97193, + "87": 1.65401, + "88": 0.99392, + "89": 0.92202, + "90": 1.19312, + "91": 0.89414, + "92": 1.55823, + "93": 0.92684, + "94": 1.1219, + "95": 1.04074, + "96": 1.00989, + "97": 1.03597, + "98": 1.18879, + "99": 1.65464, + "100": 1.0402 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json index 89891d82161..50d0de00516 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89618, - "2": 10.88407, - "3": 10.88374, - "4": 10.87907, - "5": 10.88134, - "6": 10.87733, - "7": 10.8817, - "8": 10.87724, - "9": 10.88198, - "10": 10.87794, - "11": 10.87487, - "12": 10.87061, - "13": 10.86692, - "14": 10.86694, - "15": 10.8373, - "16": 10.8338, - "17": 10.84812, - "18": 10.81884, - "19": 10.84081, - "20": 10.75972, - "21": 10.74725, - "22": 10.73529, - "23": 10.7274, - "24": 10.69964, - "25": 10.6946, - "26": 10.68328, - "27": 10.65159, - "28": 10.58828, - "29": 10.55162, - "30": 10.5269, - "31": 10.52334, - "32": 10.5136, - "33": 10.47625, - "34": 10.45172, - "35": 10.4505, - "36": 10.42348, - "37": 10.39367, - "38": 10.39639, - "39": 10.36572, - "40": 10.35672, - "41": 10.32964, - "42": 10.31025, - "43": 10.2917, - "44": 10.26514, - "45": 10.27352, - "46": 10.24067, - "47": 10.22491, - "48": 10.18093, - "49": 10.17992, - "50": 10.18057 + "1": 10.89622, + "2": 10.88399, + "3": 10.88377, + "4": 10.87916, + "5": 10.88139, + "6": 10.87726, + "7": 10.88177, + "8": 10.87729, + "9": 10.88194, + "10": 10.87806, + "11": 10.87497, + "12": 10.87067, + "13": 10.86698, + "14": 10.86691, + "15": 10.83738, + "16": 10.8339, + "17": 10.84808, + "18": 10.81879, + "19": 10.84086, + "20": 10.75964, + "21": 10.74722, + "22": 10.73534, + "23": 10.72735, + "24": 10.69971, + "25": 10.69462, + "26": 10.68333, + "27": 10.65162, + "28": 10.5883, + "29": 10.55169, + "30": 10.52704, + "31": 10.52342, + "32": 10.51368, + "33": 10.47637, + "34": 10.4518, + "35": 10.45054, + "36": 10.42361, + "37": 10.39366, + "38": 10.39646, + "39": 10.36567, + "40": 10.35674, + "41": 10.32979, + "42": 10.31027, + "43": 10.29172, + "44": 10.2652, + "45": 10.27355, + "46": 10.24076, + "47": 10.22498, + "48": 10.18094, + "49": 10.17991, + "50": 10.18064 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1762.0, + "1": 1777.0, "2": 1889.0, - "3": 1900.0, - "4": 1800.0, - "5": 1789.0, - "6": 1845.0, - "7": 1864.0, - "8": 1730.0, - "9": 1858.0, - "10": 1800.0, - "11": 1743.0, - "12": 1830.0, - "13": 1774.0, - "14": 1940.0, - "15": 1660.0, - "16": 1784.0, + "3": 1821.0, + "4": 1785.0, + "5": 1822.0, + "6": 1847.0, + "7": 1942.0, + "8": 1725.0, + "9": 1853.0, + "10": 1927.0, + "11": 1698.0, + "12": 1947.0, + "13": 1768.0, + "14": 1860.0, + "15": 1746.0, + "16": 1839.0, "17": 1818.0, - "18": 1799.0, - "19": 1764.0, - "20": 1708.0, - "21": 1809.0, - "22": 1791.0, - "23": 1857.0, - "24": 1823.0, - "25": 1767.0, - "26": 1807.0, - "27": 1733.0, + "18": 1773.0, + "19": 1717.0, + "20": 1716.0, + "21": 1843.0, + "22": 1769.0, + "23": 1859.0, + "24": 1892.0, + "25": 1826.0, + "26": 1887.0, + "27": 1799.0, "28": 1777.0, - "29": 1936.0, - "30": 1825.0, - "31": 2069.0, - "32": 2079.0, - "33": 1906.0, - "34": 2055.0, - "35": 2094.0, - "36": 2008.0, - "37": 2222.0, - "38": 2144.0, - "39": 2140.0, - "40": 2369.0, - "41": 2430.0, - "42": 2066.0, - "43": 2385.0, - "44": 2286.0, - "45": 2652.0, - "46": 2584.0, - "47": 2584.0, - "48": 2677.0, - "49": 2938.0, - "50": 2620.0 + "29": 1934.0, + "30": 1851.0, + "31": 2022.0, + "32": 2016.0, + "33": 1841.0, + "34": 2024.0, + "35": 2120.0, + "36": 1988.0, + "37": 2187.0, + "38": 2107.0, + "39": 2166.0, + "40": 2334.0, + "41": 2387.0, + "42": 1955.0, + "43": 2467.0, + "44": 2254.0, + "45": 2683.0, + "46": 2555.0, + "47": 2451.0, + "48": 2692.0, + "49": 3067.0, + "50": 2498.0 } }, "mem-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1480436736.0, - "2": 1542892032.0, - "3": 1542892032.0, - "4": 1542892032.0, - "5": 1542892032.0, - "6": 1542892032.0, - "7": 1542892032.0, - "8": 1542892032.0, - "9": 1542892032.0, - "10": 1542892032.0, - "11": 1542892032.0, - "12": 1542892032.0, - "13": 1542892032.0, - "14": 1542892032.0, - "15": 1542892032.0, - "16": 1542892032.0, - "17": 1542892032.0, - "18": 1542892032.0, - "19": 1542892032.0, - "20": 1542892032.0, - "21": 1542892032.0, - "22": 1542892032.0, - "23": 1542892032.0, - "24": 1542892032.0, - "25": 1542892032.0, - "26": 1542892032.0, - "27": 1542892032.0, - "28": 1542892032.0, - "29": 1542892032.0, - "30": 1542892032.0, - "31": 1542892032.0, - "32": 1542892032.0, - "33": 1542892032.0, - "34": 1542892032.0, - "35": 1542892032.0, - "36": 1542892032.0, - "37": 1542892032.0, - "38": 1542892032.0, - "39": 1542892032.0, - "40": 1542892032.0, - "41": 1542892032.0, - "42": 1542892032.0, - "43": 1542892032.0, - "44": 1542892032.0, - "45": 1542892032.0, - "46": 1542892032.0, - "47": 1542892032.0, - "48": 1542892032.0, - "49": 1542892032.0, - "50": 1542892032.0 + "1": 1476898304.0, + "2": 1543416320.0, + "3": 1543416320.0, + "4": 1543416320.0, + "5": 1543416320.0, + "6": 1543416320.0, + "7": 1543416320.0, + "8": 1543416320.0, + "9": 1543416320.0, + "10": 1543416320.0, + "11": 1543416320.0, + "12": 1543416320.0, + "13": 1543416320.0, + "14": 1543416320.0, + "15": 1543416320.0, + "16": 1543416320.0, + "17": 1543416320.0, + "18": 1543416320.0, + "19": 1543416320.0, + "20": 1543416320.0, + "21": 1543416320.0, + "22": 1543416320.0, + "23": 1543416320.0, + "24": 1543416320.0, + "25": 1543416320.0, + "26": 1543416320.0, + "27": 1543416320.0, + "28": 1543416320.0, + "29": 1543416320.0, + "30": 1543416320.0, + "31": 1543416320.0, + "32": 1543416320.0, + "33": 1543416320.0, + "34": 1543416320.0, + "35": 1543416320.0, + "36": 1543416320.0, + "37": 1543416320.0, + "38": 1543416320.0, + "39": 1543416320.0, + "40": 1543416320.0, + "41": 1543416320.0, + "42": 1543416320.0, + "43": 1543416320.0, + "44": 1543416320.0, + "45": 1543416320.0, + "46": 1543416320.0, + "47": 1543416320.0, + "48": 1543416320.0, + "49": 1543416320.0, + "50": 1543416320.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.55253, - "3": 0.44374, - "4": 0.36221, - "5": 0.35975, - "6": 0.36077, - "7": 0.36459, - "8": 0.36414, - "9": 0.36368, - "10": 0.36368, - "11": 0.36097, - "12": 0.36319, - "13": 0.3583, - "14": 0.36194, - "15": 0.35954, - "16": 0.40631, - "17": 0.46119, - "18": 0.45899, - "19": 0.53128, - "20": 0.47028, - "21": 0.49614, - "22": 0.42716, - "23": 0.36162, - "24": 0.36436, - "25": 0.36245, - "26": 0.35994, - "27": 0.36049, - "28": 0.35945, - "29": 0.36622, - "30": 0.36379, - "31": 0.36123, - "32": 0.36497, - "33": 0.36346, - "34": 0.36284, - "35": 0.362, - "36": 0.36384, - "37": 0.36238, - "38": 0.36178, - "39": 0.36163, - "40": 0.36177, - "41": 0.3632, - "42": 0.36443, - "43": 0.36238, - "44": 0.36249, - "45": 0.36146, - "46": 0.36357, - "47": 0.36057, - "48": 0.36628, - "49": 0.36328, - "50": 0.3616 + "2": 5.80401, + "3": 0.35254, + "4": 0.31247, + "5": 0.31253, + "6": 0.31316, + "7": 0.31332, + "8": 0.31321, + "9": 0.31822, + "10": 0.31514, + "11": 0.31386, + "12": 0.31409, + "13": 0.31271, + "14": 0.31271, + "15": 0.31555, + "16": 0.31131, + "17": 0.30609, + "18": 0.30715, + "19": 0.30738, + "20": 0.30549, + "21": 0.30525, + "22": 0.30611, + "23": 0.30818, + "24": 0.30867, + "25": 0.30935, + "26": 0.31031, + "27": 0.31036, + "28": 0.30958, + "29": 0.31054, + "30": 0.3089, + "31": 0.30911, + "32": 0.30873, + "33": 0.30741, + "34": 0.30816, + "35": 0.30986, + "36": 0.30966, + "37": 0.30953, + "38": 0.30684, + "39": 0.30882, + "40": 0.30786, + "41": 0.30962, + "42": 0.30856, + "43": 0.30693, + "44": 0.30948, + "45": 0.31025, + "46": 0.30899, + "47": 0.30973, + "48": 0.31409, + "49": 0.30992, + "50": 0.41318 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json index 4006fd0f4b8..f8c9cb3e202 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89756, - "2": 10.8907, - "3": 10.88309, - "4": 10.88974, + "1": 10.89755, + "2": 10.89063, + "3": 10.88312, + "4": 10.8897, "5": 10.88254, - "6": 10.88652, - "7": 10.88675, - "8": 10.88256, - "9": 10.88485, - "10": 10.88068, - "11": 10.87902, - "12": 10.87771, - "13": 10.87243, - "14": 10.86853, - "15": 10.84593, - "16": 10.83855, - "17": 10.83759, - "18": 10.82719, - "19": 10.83322, - "20": 10.76014, - "21": 10.74126, + "6": 10.8864, + "7": 10.8867, + "8": 10.88255, + "9": 10.88479, + "10": 10.88064, + "11": 10.87912, + "12": 10.87777, + "13": 10.87248, + "14": 10.86852, + "15": 10.84599, + "16": 10.83857, + "17": 10.83762, + "18": 10.8271, + "19": 10.83323, + "20": 10.76028, + "21": 10.74135, "22": 10.72492, "23": 10.71936, - "24": 10.70189, - "25": 10.69474, - "26": 10.6776, - "27": 10.65133, - "28": 10.57392, - "29": 10.54991, - "30": 10.52269, - "31": 10.52536, - "32": 10.5134, - "33": 10.46553, - "34": 10.4375, - "35": 10.4374, - "36": 10.42118, - "37": 10.38935, - "38": 10.38941, - "39": 10.35682, - "40": 10.34743, - "41": 10.32252, - "42": 10.30529, - "43": 10.2837, - "44": 10.26289, - "45": 10.26626, - "46": 10.23668, - "47": 10.2212, - "48": 10.17262, - "49": 10.18153, - "50": 10.17787 + "24": 10.70195, + "25": 10.69471, + "26": 10.67763, + "27": 10.65143, + "28": 10.57397, + "29": 10.5501, + "30": 10.52272, + "31": 10.52542, + "32": 10.51347, + "33": 10.46558, + "34": 10.43754, + "35": 10.43746, + "36": 10.42129, + "37": 10.38943, + "38": 10.3895, + "39": 10.35679, + "40": 10.34755, + "41": 10.32261, + "42": 10.30542, + "43": 10.28368, + "44": 10.26293, + "45": 10.2663, + "46": 10.23671, + "47": 10.2213, + "48": 10.17266, + "49": 10.18158, + "50": 10.17791 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1841.0, - "2": 1782.0, - "3": 1741.0, - "4": 1707.0, - "5": 1875.0, - "6": 1802.0, - "7": 1934.0, - "8": 1730.0, - "9": 1768.0, - "10": 1790.0, - "11": 1717.0, - "12": 1854.0, - "13": 1795.0, - "14": 1901.0, - "15": 1698.0, - "16": 1856.0, - "17": 1882.0, - "18": 1857.0, - "19": 1650.0, - "20": 1778.0, - "21": 1857.0, - "22": 1827.0, - "23": 1781.0, - "24": 1845.0, - "25": 1818.0, - "26": 1877.0, - "27": 1887.0, - "28": 1888.0, + "1": 1832.0, + "2": 1769.0, + "3": 1815.0, + "4": 1867.0, + "5": 1882.0, + "6": 1755.0, + "7": 1962.0, + "8": 1704.0, + "9": 1694.0, + "10": 1718.0, + "11": 1745.0, + "12": 1800.0, + "13": 1834.0, + "14": 1964.0, + "15": 1680.0, + "16": 1817.0, + "17": 1824.0, + "18": 1753.0, + "19": 1666.0, + "20": 1759.0, + "21": 1762.0, + "22": 1742.0, + "23": 1825.0, + "24": 1895.0, + "25": 1666.0, + "26": 1897.0, + "27": 1828.0, + "28": 1800.0, "29": 1973.0, - "30": 1819.0, - "31": 2013.0, - "32": 2072.0, - "33": 2021.0, - "34": 2116.0, - "35": 2134.0, - "36": 2021.0, - "37": 2298.0, - "38": 2052.0, - "39": 2245.0, - "40": 2299.0, - "41": 2318.0, - "42": 2073.0, - "43": 2457.0, - "44": 2132.0, - "45": 2639.0, - "46": 2424.0, - "47": 2491.0, - "48": 2650.0, - "49": 2923.0, - "50": 2666.0 + "30": 1801.0, + "31": 1983.0, + "32": 1978.0, + "33": 1990.0, + "34": 2022.0, + "35": 2120.0, + "36": 2058.0, + "37": 2236.0, + "38": 2159.0, + "39": 2242.0, + "40": 2441.0, + "41": 2363.0, + "42": 2012.0, + "43": 2325.0, + "44": 2258.0, + "45": 2529.0, + "46": 2383.0, + "47": 2468.0, + "48": 2741.0, + "49": 2977.0, + "50": 2613.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.34616, - "3": 0.3142, - "4": 0.31192, - "5": 0.31604, - "6": 0.3111, - "7": 0.31244, - "8": 0.30946, - "9": 0.31192, - "10": 0.30859, - "11": 0.30942, - "12": 0.3117, - "13": 0.31095, - "14": 0.30866, - "15": 0.30981, - "16": 0.31245, - "17": 0.31087, - "18": 0.31053, - "19": 0.30871, - "20": 0.30939, - "21": 0.3118, - "22": 0.31199, - "23": 0.30966, - "24": 0.31088, - "25": 0.31307, - "26": 0.31034, - "27": 0.31087, - "28": 0.30973, - "29": 0.30976, - "30": 0.30982, - "31": 0.30984, - "32": 0.31107, - "33": 0.30863, - "34": 0.31312, - "35": 0.30891, - "36": 0.31222, - "37": 0.31222, - "38": 0.31082, - "39": 0.31106, - "40": 0.31, - "41": 0.30893, - "42": 0.30983, - "43": 0.315, - "44": 0.31091, - "45": 0.31262, - "46": 0.30891, - "47": 0.31182, - "48": 0.30917, - "49": 0.31196, - "50": 0.31219 + "2": 3.52184, + "3": 0.33816, + "4": 0.33967, + "5": 0.33557, + "6": 0.33562, + "7": 0.34298, + "8": 0.33573, + "9": 0.33736, + "10": 0.33704, + "11": 0.33888, + "12": 0.33582, + "13": 0.33617, + "14": 0.33561, + "15": 0.33383, + "16": 0.33235, + "17": 0.33266, + "18": 0.33567, + "19": 0.33475, + "20": 0.33263, + "21": 0.33833, + "22": 0.33485, + "23": 0.33444, + "24": 0.33288, + "25": 0.33452, + "26": 0.33589, + "27": 0.33849, + "28": 0.33321, + "29": 0.33545, + "30": 0.33349, + "31": 0.3336, + "32": 0.33499, + "33": 0.33692, + "34": 0.33469, + "35": 0.33454, + "36": 0.33539, + "37": 0.33347, + "38": 0.33268, + "39": 0.33262, + "40": 0.33792, + "41": 0.33901, + "42": 0.33349, + "43": 0.33469, + "44": 0.33452, + "45": 0.33408, + "46": 0.33262, + "47": 0.33185, + "48": 0.33284, + "49": 0.33789, + "50": 0.33614 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json index 63c5bf23448..b6498c7c69e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89465, + "1": 10.89449, "2": 10.88327, - "3": 10.88075, - "4": 10.88361, - "5": 10.88287, - "6": 10.875, - "7": 10.87677, - "8": 10.88171, - "9": 10.88078, - "10": 10.87893, - "11": 10.87526, - "12": 10.86941, - "13": 10.86653, - "14": 10.86405, - "15": 10.8404, - "16": 10.83432, - "17": 10.84502, - "18": 10.81955, - "19": 10.84023, - "20": 10.75963, - "21": 10.74833, - "22": 10.73542, - "23": 10.72861, - "24": 10.70175, - "25": 10.69385, - "26": 10.68277, - "27": 10.65028, - "28": 10.58845, - "29": 10.5527, - "30": 10.53012, - "31": 10.52065, - "32": 10.51506, + "3": 10.88077, + "4": 10.88364, + "5": 10.8829, + "6": 10.87498, + "7": 10.8768, + "8": 10.88173, + "9": 10.88076, + "10": 10.87889, + "11": 10.87524, + "12": 10.86948, + "13": 10.86656, + "14": 10.86411, + "15": 10.84044, + "16": 10.8343, + "17": 10.8452, + "18": 10.81964, + "19": 10.84026, + "20": 10.75966, + "21": 10.74836, + "22": 10.73543, + "23": 10.72868, + "24": 10.70162, + "25": 10.69393, + "26": 10.68276, + "27": 10.6503, + "28": 10.58853, + "29": 10.55281, + "30": 10.53027, + "31": 10.52072, + "32": 10.51514, "33": 10.47611, - "34": 10.45066, - "35": 10.45166, - "36": 10.42459, - "37": 10.39344, - "38": 10.39768, - "39": 10.36673, - "40": 10.35691, - "41": 10.33049, - "42": 10.30995, - "43": 10.28971, - "44": 10.26685, - "45": 10.27231, - "46": 10.23941, - "47": 10.2249, - "48": 10.18112, - "49": 10.1825, - "50": 10.18095 + "34": 10.45072, + "35": 10.45162, + "36": 10.42463, + "37": 10.39354, + "38": 10.39778, + "39": 10.36681, + "40": 10.35697, + "41": 10.33059, + "42": 10.31008, + "43": 10.28977, + "44": 10.26692, + "45": 10.27237, + "46": 10.23947, + "47": 10.22497, + "48": 10.18114, + "49": 10.18252, + "50": 10.18097 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1815.0, - "2": 1778.0, - "3": 1884.0, - "4": 1723.0, - "5": 1861.0, - "6": 1834.0, - "7": 1939.0, - "8": 1787.0, - "9": 1863.0, - "10": 1827.0, - "11": 1824.0, - "12": 1776.0, - "13": 1751.0, - "14": 1921.0, - "15": 1778.0, - "16": 1802.0, - "17": 1812.0, - "18": 1776.0, - "19": 1781.0, - "20": 1731.0, - "21": 1944.0, - "22": 1875.0, - "23": 1834.0, - "24": 1839.0, - "25": 1770.0, - "26": 1904.0, - "27": 1826.0, - "28": 1782.0, - "29": 1928.0, - "30": 1923.0, - "31": 2037.0, - "32": 1915.0, - "33": 1951.0, - "34": 2059.0, - "35": 2146.0, - "36": 2031.0, - "37": 2301.0, - "38": 2103.0, - "39": 2248.0, - "40": 2226.0, - "41": 2352.0, - "42": 1925.0, - "43": 2435.0, - "44": 2241.0, - "45": 2601.0, - "46": 2495.0, - "47": 2557.0, - "48": 2705.0, - "49": 2785.0, - "50": 2545.0 + "1": 1835.0, + "2": 1740.0, + "3": 1915.0, + "4": 1845.0, + "5": 1801.0, + "6": 1751.0, + "7": 1964.0, + "8": 1798.0, + "9": 1884.0, + "10": 1854.0, + "11": 1807.0, + "12": 1799.0, + "13": 1853.0, + "14": 1976.0, + "15": 1714.0, + "16": 1875.0, + "17": 1860.0, + "18": 1818.0, + "19": 1757.0, + "20": 1785.0, + "21": 1831.0, + "22": 1790.0, + "23": 1849.0, + "24": 1850.0, + "25": 1769.0, + "26": 1857.0, + "27": 1883.0, + "28": 1851.0, + "29": 1971.0, + "30": 1913.0, + "31": 2028.0, + "32": 1980.0, + "33": 2012.0, + "34": 2034.0, + "35": 2228.0, + "36": 1947.0, + "37": 2153.0, + "38": 2099.0, + "39": 2219.0, + "40": 2291.0, + "41": 2282.0, + "42": 2009.0, + "43": 2445.0, + "44": 2251.0, + "45": 2616.0, + "46": 2503.0, + "47": 2614.0, + "48": 2743.0, + "49": 2913.0, + "50": 2590.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21165, - "3": 2.4816, - "4": 2.18761, - "5": 1.77345, - "6": 2.25446, - "7": 1.92959, - "8": 2.58115, - "9": 2.62203, - "10": 2.16926, - "11": 2.30418, - "12": 1.94962, - "13": 2.26072, - "14": 2.32385, - "15": 1.88242, - "16": 2.10012, - "17": 2.7401, - "18": 1.92322, - "19": 2.32259, - "20": 1.92946, - "21": 2.42083, - "22": 1.79961, - "23": 2.28145, - "24": 1.79815, - "25": 2.42588, - "26": 2.61199, - "27": 1.90333, - "28": 2.00642, - "29": 1.73461, - "30": 1.99201, - "31": 1.7887, - "32": 3.05861, - "33": 1.76916, - "34": 1.65386, - "35": 2.44925, - "36": 2.34312, - "37": 1.98498, - "38": 1.83213, - "39": 2.40965, - "40": 2.02627, - "41": 2.17662, - "42": 2.70897, - "43": 1.89561, - "44": 2.08977, - "45": 2.63009, - "46": 1.8783, - "47": 2.20647, - "48": 2.38399, - "49": 1.79461, - "50": 2.29719 + "2": 5.0734, + "3": 2.45055, + "4": 2.0712, + "5": 1.70873, + "6": 2.13591, + "7": 1.71649, + "8": 2.41206, + "9": 2.55111, + "10": 2.12984, + "11": 2.20278, + "12": 1.92149, + "13": 2.13738, + "14": 2.30029, + "15": 1.78388, + "16": 1.98995, + "17": 2.56746, + "18": 1.95678, + "19": 2.17153, + "20": 1.9358, + "21": 2.31315, + "22": 1.76007, + "23": 2.24781, + "24": 1.81786, + "25": 2.43349, + "26": 2.56395, + "27": 1.88577, + "28": 2.0396, + "29": 1.6388, + "30": 2.01431, + "31": 1.70931, + "32": 2.92669, + "33": 1.61913, + "34": 1.59961, + "35": 2.59046, + "36": 2.09439, + "37": 2.03004, + "38": 1.62614, + "39": 2.45007, + "40": 2.0193, + "41": 2.13522, + "42": 2.69777, + "43": 1.87251, + "44": 2.02357, + "45": 2.53806, + "46": 1.88318, + "47": 2.3303, + "48": 2.52999, + "49": 1.92962, + "50": 2.69212 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json index fa603bfe9ad..623ee5e0b68 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.89594, - "2": 10.88351, - "3": 10.88336, - "4": 10.87907, - "5": 10.88164, - "6": 10.87691, - "7": 10.88192, - "8": 10.87682, - "9": 10.88197, - "10": 10.87748, - "11": 10.87492, - "12": 10.87082, - "13": 10.86703, - "14": 10.86686, - "15": 10.83756, + "2": 10.88348, + "3": 10.88337, + "4": 10.87919, + "5": 10.88173, + "6": 10.87681, + "7": 10.88197, + "8": 10.87683, + "9": 10.88199, + "10": 10.87754, + "11": 10.87498, + "12": 10.8708, + "13": 10.86709, + "14": 10.86685, + "15": 10.83763, "16": 10.83405, - "17": 10.84806, - "18": 10.819, - "19": 10.8413, + "17": 10.84804, + "18": 10.81896, + "19": 10.84141, "20": 10.75952, - "21": 10.74748, - "22": 10.73572, - "23": 10.72717, - "24": 10.69979, - "25": 10.69416, - "26": 10.68361, - "27": 10.65164, - "28": 10.58846, - "29": 10.55137, - "30": 10.52741, - "31": 10.52354, - "32": 10.51393, - "33": 10.47623, - "34": 10.45224, - "35": 10.45087, - "36": 10.42396, - "37": 10.39417, - "38": 10.39666, + "21": 10.74743, + "22": 10.73574, + "23": 10.72715, + "24": 10.69988, + "25": 10.69423, + "26": 10.68366, + "27": 10.65169, + "28": 10.58848, + "29": 10.55149, + "30": 10.52757, + "31": 10.52361, + "32": 10.51402, + "33": 10.47633, + "34": 10.45235, + "35": 10.45092, + "36": 10.42406, + "37": 10.39411, + "38": 10.39675, "39": 10.36578, - "40": 10.35693, - "41": 10.33, - "42": 10.31048, - "43": 10.29194, + "40": 10.35697, + "41": 10.3301, + "42": 10.31054, + "43": 10.29193, "44": 10.26509, - "45": 10.27365, - "46": 10.24094, - "47": 10.22528, - "48": 10.18097, + "45": 10.27371, + "46": 10.241, + "47": 10.22535, + "48": 10.18099, "49": 10.18011, - "50": 10.18079, - "51": 10.18221, - "52": 10.13889, - "53": 10.14365, - "54": 10.10726, - "55": 10.08316, - "56": 10.10805, - "57": 10.09842, - "58": 10.11326, - "59": 10.05956, - "60": 10.07989, - "61": 10.02737, - "62": 10.00023, - "63": 10.07516, - "64": 10.03039, - "65": 10.00784, - "66": 10.02965, - "67": 10.00776, - "68": 9.9695, - "69": 9.99311, - "70": 9.96958, - "71": 9.99891, - "72": 9.98004, - "73": 9.97026, - "74": 9.95555, - "75": 9.93141, - "76": 9.96391, - "77": 9.95308, + "50": 10.18084, + "51": 10.18228, + "52": 10.13893, + "53": 10.14375, + "54": 10.10729, + "55": 10.08325, + "56": 10.10813, + "57": 10.0984, + "58": 10.11324, + "59": 10.0596, + "60": 10.07993, + "61": 10.02738, + "62": 10.00025, + "63": 10.07518, + "64": 10.03038, + "65": 10.00788, + "66": 10.02969, + "67": 10.00781, + "68": 9.96955, + "69": 9.99313, + "70": 9.96964, + "71": 9.99895, + "72": 9.98009, + "73": 9.9703, + "74": 9.9556, + "75": 9.93146, + "76": 9.96394, + "77": 9.95318, "78": 9.90667, - "79": 9.91319, - "80": 9.92642, - "81": 9.95383, - "82": 9.88691, - "83": 9.85299, - "84": 9.7942, - "85": 9.7874, - "86": 9.87886, - "87": 9.90278, - "88": 9.87661, - "89": 9.82312, - "90": 9.81092, - "91": 9.82769, - "92": 9.81636, - "93": 9.75384, - "94": 9.82164, - "95": 9.8208, - "96": 9.80275, - "97": 9.74807, - "98": 9.77578, - "99": 9.81931, - "100": 9.71179 + "79": 9.91324, + "80": 9.92648, + "81": 9.95394, + "82": 9.88695, + "83": 9.85307, + "84": 9.79427, + "85": 9.78745, + "86": 9.87895, + "87": 9.90281, + "88": 9.87662, + "89": 9.82317, + "90": 9.81101, + "91": 9.82773, + "92": 9.81639, + "93": 9.75392, + "94": 9.82167, + "95": 9.82079, + "96": 9.80272, + "97": 9.74813, + "98": 9.77586, + "99": 9.81936, + "100": 9.71183 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1785.0, - "3": 1867.0, - "4": 1822.0, - "5": 1837.0, - "6": 1806.0, - "7": 1844.0, - "8": 1819.0, - "9": 1847.0, - "10": 1793.0, - "11": 1697.0, - "12": 1730.0, - "13": 1876.0, - "14": 1976.0, - "15": 1711.0, - "16": 1732.0, - "17": 1877.0, - "18": 1704.0, - "19": 1753.0, - "20": 1742.0, - "21": 1870.0, - "22": 1737.0, - "23": 1766.0, - "24": 1894.0, - "25": 1748.0, - "26": 1901.0, - "27": 1796.0, - "28": 1770.0, - "29": 2038.0, - "30": 1937.0, - "31": 2021.0, - "32": 1948.0, - "33": 1912.0, - "34": 2066.0, - "35": 2170.0, - "36": 2011.0, - "37": 2269.0, - "38": 2148.0, - "39": 2220.0, - "40": 2207.0, - "41": 2356.0, - "42": 2028.0, - "43": 2464.0, - "44": 2269.0, - "45": 2652.0, - "46": 2465.0, - "47": 2469.0, - "48": 2636.0, - "49": 3091.0, - "50": 2552.0, - "51": 2525.0, - "52": 2776.0, - "53": 2568.0, - "54": 3007.0, - "55": 2615.0, - "56": 2863.0, - "57": 2162.0, - "58": 3765.0, - "59": 3016.0, - "60": 3097.0, - "61": 2923.0, - "62": 3282.0, - "63": 3397.0, - "64": 3620.0, - "65": 2761.0, - "66": 3135.0, - "67": 4026.0, - "68": 3404.0, - "69": 3077.0, - "70": 3356.0, - "71": 3083.0, - "72": 2995.0, - "73": 3373.0, - "74": 3400.0, - "75": 3234.0, - "76": 3187.0, - "77": 3710.0, - "78": 3325.0, - "79": 3291.0, - "80": 3109.0, - "81": 3546.0, - "82": 2804.0, - "83": 3015.0, - "84": 2944.0, - "85": 2690.0, - "86": 3183.0, - "87": 2848.0, + "1": 1788.0, + "2": 1822.0, + "3": 1934.0, + "4": 1751.0, + "5": 1817.0, + "6": 1901.0, + "7": 1871.0, + "8": 1776.0, + "9": 1843.0, + "10": 1866.0, + "11": 1754.0, + "12": 1771.0, + "13": 1844.0, + "14": 1888.0, + "15": 1696.0, + "16": 1811.0, + "17": 1801.0, + "18": 1844.0, + "19": 1711.0, + "20": 1770.0, + "21": 1899.0, + "22": 1771.0, + "23": 1791.0, + "24": 1896.0, + "25": 1758.0, + "26": 1844.0, + "27": 1846.0, + "28": 1744.0, + "29": 1985.0, + "30": 1880.0, + "31": 1993.0, + "32": 1965.0, + "33": 2010.0, + "34": 2169.0, + "35": 2121.0, + "36": 2051.0, + "37": 2232.0, + "38": 2191.0, + "39": 2258.0, + "40": 2300.0, + "41": 2373.0, + "42": 2002.0, + "43": 2479.0, + "44": 2220.0, + "45": 2714.0, + "46": 2570.0, + "47": 2470.0, + "48": 2678.0, + "49": 2903.0, + "50": 2547.0, + "51": 2596.0, + "52": 2799.0, + "53": 2707.0, + "54": 2912.0, + "55": 2646.0, + "56": 2801.0, + "57": 2307.0, + "58": 3827.0, + "59": 3124.0, + "60": 3010.0, + "61": 2888.0, + "62": 3249.0, + "63": 3314.0, + "64": 3700.0, + "65": 2838.0, + "66": 3203.0, + "67": 4095.0, + "68": 3537.0, + "69": 3118.0, + "70": 3438.0, + "71": 3227.0, + "72": 3153.0, + "73": 3391.0, + "74": 3411.0, + "75": 3190.0, + "76": 3260.0, + "77": 3684.0, + "78": 3334.0, + "79": 3304.0, + "80": 3121.0, + "81": 3512.0, + "82": 2796.0, + "83": 3087.0, + "84": 2906.0, + "85": 2724.0, + "86": 3157.0, + "87": 2947.0, "88": 3148.0, - "89": 2937.0, - "90": 3994.0, - "91": 3014.0, + "89": 3035.0, + "90": 4042.0, + "91": 2929.0, "92": 3084.0, - "93": 3130.0, - "94": 3009.0, - "95": 3348.0, - "96": 3626.0, - "97": 3678.0, - "98": 3523.0, - "99": 3262.0, - "100": 3306.0 + "93": 3015.0, + "94": 3145.0, + "95": 3337.0, + "96": 3515.0, + "97": 3722.0, + "98": 3495.0, + "99": 3328.0, + "100": 3390.0 } }, "mem-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 999540224.0, - "2": 1065140736.0, - "3": 1065140736.0, - "4": 1065140736.0, - "5": 1065140736.0, - "6": 1065140736.0, - "7": 1065140736.0, - "8": 1065140736.0, - "9": 1065140736.0, - "10": 1065140736.0, - "11": 1065140736.0, - "12": 1065140736.0, - "13": 1065140736.0, - "14": 1065140736.0, - "15": 1065140736.0, - "16": 1065140736.0, - "17": 1065140736.0, - "18": 1065140736.0, - "19": 1065140736.0, - "20": 1065140736.0, - "21": 1065140736.0, - "22": 1065140736.0, - "23": 1065140736.0, - "24": 1065140736.0, - "25": 1065140736.0, - "26": 1065140736.0, - "27": 1065140736.0, - "28": 1065140736.0, - "29": 1065140736.0, - "30": 1065140736.0, - "31": 1065140736.0, - "32": 1065140736.0, - "33": 1065140736.0, - "34": 1065140736.0, - "35": 1065140736.0, - "36": 1065140736.0, - "37": 1065140736.0, - "38": 1065140736.0, - "39": 1065140736.0, - "40": 1065140736.0, - "41": 1065140736.0, - "42": 1065140736.0, - "43": 1065140736.0, - "44": 1065140736.0, - "45": 1065140736.0, - "46": 1065140736.0, - "47": 1065140736.0, - "48": 1065140736.0, - "49": 1065140736.0, - "50": 1065140736.0, - "51": 1065140736.0, - "52": 1065140736.0, - "53": 1065140736.0, - "54": 1065140736.0, - "55": 1065140736.0, - "56": 1065140736.0, - "57": 1065140736.0, - "58": 1065140736.0, - "59": 1065140736.0, - "60": 1065140736.0, - "61": 1065140736.0, - "62": 1065140736.0, - "63": 1065140736.0, - "64": 1065140736.0, - "65": 1065140736.0, - "66": 1065140736.0, - "67": 1065140736.0, - "68": 1065140736.0, - "69": 1065140736.0, - "70": 1065140736.0, - "71": 1065140736.0, - "72": 1065140736.0, - "73": 1065140736.0, - "74": 1065140736.0, - "75": 1065140736.0, - "76": 1065140736.0, - "77": 1065140736.0, - "78": 1065140736.0, - "79": 1065140736.0, - "80": 1065140736.0, - "81": 1065140736.0, - "82": 1065140736.0, - "83": 1065140736.0, - "84": 1065140736.0, - "85": 1065140736.0, - "86": 1065140736.0, - "87": 1065140736.0, - "88": 1065140736.0, - "89": 1065140736.0, - "90": 1065140736.0, - "91": 1065140736.0, - "92": 1065140736.0, - "93": 1065140736.0, - "94": 1065140736.0, - "95": 1065140736.0, - "96": 1065140736.0, - "97": 1065140736.0, - "98": 1065140736.0, - "99": 1065140736.0, - "100": 1065140736.0 + "1": 995345920.0, + "2": 1059371008.0, + "3": 1059371008.0, + "4": 1059371008.0, + "5": 1059371008.0, + "6": 1059371008.0, + "7": 1059371008.0, + "8": 1059371008.0, + "9": 1059371008.0, + "10": 1059371008.0, + "11": 1059371008.0, + "12": 1059371008.0, + "13": 1059371008.0, + "14": 1059371008.0, + "15": 1059371008.0, + "16": 1059371008.0, + "17": 1059371008.0, + "18": 1059371008.0, + "19": 1059371008.0, + "20": 1059371008.0, + "21": 1059371008.0, + "22": 1059371008.0, + "23": 1059371008.0, + "24": 1059371008.0, + "25": 1059371008.0, + "26": 1059371008.0, + "27": 1059371008.0, + "28": 1059371008.0, + "29": 1059371008.0, + "30": 1059371008.0, + "31": 1059371008.0, + "32": 1059371008.0, + "33": 1059371008.0, + "34": 1059371008.0, + "35": 1059371008.0, + "36": 1059371008.0, + "37": 1059371008.0, + "38": 1059371008.0, + "39": 1059371008.0, + "40": 1059371008.0, + "41": 1059371008.0, + "42": 1059371008.0, + "43": 1059371008.0, + "44": 1059371008.0, + "45": 1059371008.0, + "46": 1059371008.0, + "47": 1059371008.0, + "48": 1059371008.0, + "49": 1059371008.0, + "50": 1059371008.0, + "51": 1059371008.0, + "52": 1059371008.0, + "53": 1059371008.0, + "54": 1059371008.0, + "55": 1059371008.0, + "56": 1059371008.0, + "57": 1059371008.0, + "58": 1059371008.0, + "59": 1059371008.0, + "60": 1059371008.0, + "61": 1059371008.0, + "62": 1059371008.0, + "63": 1059371008.0, + "64": 1059371008.0, + "65": 1059371008.0, + "66": 1059371008.0, + "67": 1059371008.0, + "68": 1059371008.0, + "69": 1059371008.0, + "70": 1059371008.0, + "71": 1059371008.0, + "72": 1059371008.0, + "73": 1059371008.0, + "74": 1059371008.0, + "75": 1059371008.0, + "76": 1059371008.0, + "77": 1059371008.0, + "78": 1059371008.0, + "79": 1059371008.0, + "80": 1059371008.0, + "81": 1059371008.0, + "82": 1059371008.0, + "83": 1059371008.0, + "84": 1059371008.0, + "85": 1059371008.0, + "86": 1059371008.0, + "87": 1059371008.0, + "88": 1059371008.0, + "89": 1059371008.0, + "90": 1059371008.0, + "91": 1059371008.0, + "92": 1059371008.0, + "93": 1059371008.0, + "94": 1059371008.0, + "95": 1059371008.0, + "96": 1059371008.0, + "97": 1059371008.0, + "98": 1059371008.0, + "99": 1059371008.0, + "100": 1059371008.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.78137, - "3": 0.33327, - "4": 0.31937, - "5": 0.31614, - "6": 0.31454, - "7": 0.31946, - "8": 0.31642, - "9": 0.32038, - "10": 0.36794, - "11": 0.55217, - "12": 0.5196, - "13": 0.52035, - "14": 0.43681, - "15": 0.47894, - "16": 0.31481, - "17": 0.31416, - "18": 0.31394, - "19": 0.31355, - "20": 0.31487, - "21": 0.31836, - "22": 0.31682, - "23": 0.31562, - "24": 0.3151, - "25": 0.31911, - "26": 0.31535, - "27": 0.3171, - "28": 0.31442, - "29": 0.31581, - "30": 0.31835, - "31": 0.31517, - "32": 0.31506, - "33": 0.31487, - "34": 0.3154, - "35": 0.31623, - "36": 0.31642, - "37": 0.3163, - "38": 0.31654, - "39": 0.31922, - "40": 0.31672, - "41": 0.31523, - "42": 0.31475, - "43": 0.31622, - "44": 0.31716, - "45": 0.31465, - "46": 0.31613, - "47": 0.31505, - "48": 0.31368, - "49": 0.3147, - "50": 0.31696, - "51": 0.49671, - "52": 0.37055, - "53": 0.32398, - "54": 0.31658, - "55": 0.31564, - "56": 0.31843, - "57": 0.31696, - "58": 0.32084, - "59": 0.31756, - "60": 0.31705, - "61": 0.31695, - "62": 0.31692, - "63": 0.31741, - "64": 0.31886, - "65": 0.32061, - "66": 0.31656, - "67": 0.31857, - "68": 0.31788, - "69": 0.31914, - "70": 0.31951, - "71": 0.34451, - "72": 0.32937, - "73": 0.33951, - "74": 0.38335, - "75": 0.33166, - "76": 0.31673, - "77": 0.31546, - "78": 0.3157, - "79": 0.31412, - "80": 0.31417, - "81": 0.31491, - "82": 0.31476, - "83": 0.31515, - "84": 0.31558, - "85": 0.32347, - "86": 0.32021, - "87": 0.32205, - "88": 0.31973, - "89": 0.32039, - "90": 0.3197, - "91": 0.31633, - "92": 0.31657, - "93": 0.31671, - "94": 0.31536, - "95": 0.31629, - "96": 0.31628, - "97": 0.31708, - "98": 0.31867, - "99": 0.31435, - "100": 0.31525 + "2": 5.81439, + "3": 0.31572, + "4": 0.26414, + "5": 0.26631, + "6": 0.26546, + "7": 0.26423, + "8": 0.26705, + "9": 0.26657, + "10": 0.26504, + "11": 0.26695, + "12": 0.26894, + "13": 0.26914, + "14": 0.26704, + "15": 0.26745, + "16": 0.26722, + "17": 0.26814, + "18": 0.26911, + "19": 0.26804, + "20": 0.2664, + "21": 0.26733, + "22": 0.26965, + "23": 0.26543, + "24": 0.26903, + "25": 0.2677, + "26": 0.26583, + "27": 0.26827, + "28": 0.26835, + "29": 0.26801, + "30": 0.26893, + "31": 0.26823, + "32": 0.26832, + "33": 0.26878, + "34": 0.26578, + "35": 0.26857, + "36": 0.26899, + "37": 0.26622, + "38": 0.26882, + "39": 0.26699, + "40": 0.2675, + "41": 0.26802, + "42": 0.26744, + "43": 0.26722, + "44": 0.2678, + "45": 0.26774, + "46": 0.26849, + "47": 0.27113, + "48": 0.2668, + "49": 0.26615, + "50": 0.26694, + "51": 0.45071, + "52": 0.27573, + "53": 0.32127, + "54": 0.26407, + "55": 0.26652, + "56": 0.26686, + "57": 0.26315, + "58": 0.26746, + "59": 0.26975, + "60": 0.27248, + "61": 0.2694, + "62": 0.26862, + "63": 0.26727, + "64": 0.26904, + "65": 0.267, + "66": 0.26394, + "67": 0.26411, + "68": 0.26538, + "69": 0.26516, + "70": 0.26471, + "71": 0.26491, + "72": 0.26726, + "73": 0.26584, + "74": 0.26883, + "75": 0.2686, + "76": 0.26909, + "77": 0.26678, + "78": 0.26795, + "79": 0.26806, + "80": 0.26872, + "81": 0.26876, + "82": 0.27226, + "83": 0.26967, + "84": 0.273, + "85": 0.27659, + "86": 0.67951, + "87": 0.27464, + "88": 0.27415, + "89": 0.27512, + "90": 0.27783, + "91": 0.27274, + "92": 0.2737, + "93": 0.27132, + "94": 0.26958, + "95": 0.27028, + "96": 0.2735, + "97": 0.27241, + "98": 0.26858, + "99": 0.27054, + "100": 0.26862 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json index 33fbf30b29e..c2b0e29427e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89723, - "2": 10.89039, - "3": 10.88307, - "4": 10.88936, - "5": 10.88232, - "6": 10.88662, - "7": 10.88761, - "8": 10.88217, - "9": 10.8849, - "10": 10.88111, - "11": 10.87931, - "12": 10.87751, - "13": 10.87274, - "14": 10.8683, - "15": 10.84624, - "16": 10.83789, - "17": 10.83723, - "18": 10.82685, - "19": 10.83392, - "20": 10.76073, - "21": 10.74166, - "22": 10.72525, - "23": 10.71959, - "24": 10.70182, - "25": 10.69462, - "26": 10.67756, - "27": 10.65206, - "28": 10.57443, - "29": 10.5504, - "30": 10.52251, - "31": 10.5259, - "32": 10.51363, - "33": 10.46608, - "34": 10.4379, - "35": 10.43775, - "36": 10.42159, - "37": 10.38933, - "38": 10.38943, - "39": 10.3568, - "40": 10.34751, - "41": 10.32304, - "42": 10.30556, - "43": 10.28355, + "1": 10.89725, + "2": 10.89034, + "3": 10.88313, + "4": 10.88928, + "5": 10.8823, + "6": 10.88649, + "7": 10.88759, + "8": 10.88216, + "9": 10.88485, + "10": 10.88115, + "11": 10.87936, + "12": 10.87759, + "13": 10.87278, + "14": 10.86819, + "15": 10.84637, + "16": 10.83795, + "17": 10.8373, + "18": 10.82682, + "19": 10.83389, + "20": 10.76081, + "21": 10.74174, + "22": 10.72524, + "23": 10.71965, + "24": 10.70188, + "25": 10.69461, + "26": 10.67754, + "27": 10.6521, + "28": 10.57447, + "29": 10.55048, + "30": 10.52256, + "31": 10.52596, + "32": 10.51369, + "33": 10.46614, + "34": 10.43797, + "35": 10.43785, + "36": 10.42172, + "37": 10.38938, + "38": 10.38955, + "39": 10.35676, + "40": 10.34758, + "41": 10.32306, + "42": 10.30569, + "43": 10.28356, "44": 10.26322, - "45": 10.2662, - "46": 10.2368, - "47": 10.22146, - "48": 10.17272, - "49": 10.18156, - "50": 10.17791, - "51": 10.18023, - "52": 10.13377, - "53": 10.13153, - "54": 10.10638, - "55": 10.08296, - "56": 10.10637, - "57": 10.10201, + "45": 10.26623, + "46": 10.23684, + "47": 10.22155, + "48": 10.17277, + "49": 10.18161, + "50": 10.17798, + "51": 10.18021, + "52": 10.13376, + "53": 10.13158, + "54": 10.10647, + "55": 10.08302, + "56": 10.10644, + "57": 10.10207, "58": 10.10817, - "59": 10.04452, - "60": 10.07106, - "61": 10.02559, - "62": 9.9984, - "63": 10.06817, - "64": 10.01965, - "65": 9.98079, - "66": 10.02431, - "67": 10.00269, - "68": 9.96782, - "69": 9.98685, - "70": 9.96672, - "71": 9.99227, - "72": 9.97696, + "59": 10.04457, + "60": 10.07112, + "61": 10.0256, + "62": 9.99842, + "63": 10.06818, + "64": 10.01971, + "65": 9.98073, + "66": 10.02437, + "67": 10.0027, + "68": 9.96789, + "69": 9.98687, + "70": 9.96676, + "71": 9.9923, + "72": 9.97702, "73": 9.96099, - "74": 9.95636, - "75": 9.92492, - "76": 9.95108, - "77": 9.94711, - "78": 9.9007, - "79": 9.90293, - "80": 9.9174, - "81": 9.94305, - "82": 9.88166, - "83": 9.84046, - "84": 9.78038, - "85": 9.77116, - "86": 9.87352, - "87": 9.90184, - "88": 9.87564, - "89": 9.82672, - "90": 9.81115, - "91": 9.82071, - "92": 9.81382, - "93": 9.7493, - "94": 9.81784, - "95": 9.80819, - "96": 9.7946, - "97": 9.74391, - "98": 9.76747, - "99": 9.81868, - "100": 9.705 + "74": 9.95639, + "75": 9.92493, + "76": 9.95112, + "77": 9.94724, + "78": 9.90078, + "79": 9.90296, + "80": 9.91747, + "81": 9.94309, + "82": 9.88173, + "83": 9.84051, + "84": 9.78052, + "85": 9.7712, + "86": 9.87355, + "87": 9.9019, + "88": 9.87567, + "89": 9.82676, + "90": 9.81117, + "91": 9.82076, + "92": 9.81381, + "93": 9.74936, + "94": 9.81789, + "95": 9.80823, + "96": 9.79458, + "97": 9.74393, + "98": 9.76746, + "99": 9.81869, + "100": 9.70503 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1842.0, - "2": 1747.0, - "3": 1798.0, - "4": 1756.0, - "5": 1843.0, - "6": 1866.0, - "7": 1878.0, - "8": 1740.0, - "9": 1664.0, - "10": 1710.0, - "11": 1759.0, - "12": 1778.0, - "13": 1753.0, - "14": 1927.0, - "15": 1734.0, - "16": 1783.0, - "17": 1838.0, - "18": 1749.0, - "19": 1687.0, - "20": 1660.0, - "21": 1807.0, - "22": 1826.0, - "23": 1718.0, - "24": 1932.0, - "25": 1716.0, - "26": 1844.0, - "27": 1871.0, - "28": 1786.0, - "29": 1949.0, - "30": 1805.0, - "31": 2018.0, - "32": 2076.0, - "33": 1949.0, - "34": 2154.0, - "35": 2209.0, - "36": 2126.0, - "37": 2259.0, - "38": 2249.0, - "39": 2310.0, - "40": 2346.0, - "41": 2381.0, - "42": 2041.0, - "43": 2284.0, - "44": 2233.0, - "45": 2672.0, - "46": 2483.0, - "47": 2560.0, - "48": 2663.0, - "49": 2907.0, - "50": 2702.0, - "51": 2562.0, - "52": 2823.0, - "53": 2633.0, - "54": 2876.0, - "55": 2635.0, - "56": 2793.0, - "57": 2287.0, - "58": 3592.0, - "59": 3073.0, - "60": 2865.0, - "61": 2722.0, - "62": 3098.0, - "63": 3278.0, - "64": 3668.0, - "65": 2763.0, - "66": 3267.0, - "67": 3872.0, - "68": 3554.0, - "69": 3020.0, - "70": 3285.0, - "71": 3065.0, - "72": 3037.0, - "73": 3495.0, - "74": 3198.0, - "75": 3274.0, - "76": 3402.0, - "77": 3744.0, - "78": 3175.0, - "79": 3312.0, - "80": 3230.0, - "81": 3448.0, - "82": 2991.0, - "83": 3114.0, - "84": 2796.0, - "85": 2725.0, - "86": 3304.0, - "87": 2802.0, - "88": 3066.0, - "89": 2962.0, - "90": 3757.0, - "91": 3102.0, - "92": 2938.0, - "93": 3086.0, - "94": 3253.0, - "95": 3251.0, - "96": 3529.0, - "97": 3575.0, - "98": 3498.0, - "99": 3111.0, - "100": 3147.0 + "1": 1856.0, + "2": 1781.0, + "3": 1802.0, + "4": 1768.0, + "5": 1858.0, + "6": 1894.0, + "7": 1967.0, + "8": 1731.0, + "9": 1828.0, + "10": 1828.0, + "11": 1905.0, + "12": 1779.0, + "13": 1772.0, + "14": 1893.0, + "15": 1713.0, + "16": 1852.0, + "17": 1776.0, + "18": 1843.0, + "19": 1633.0, + "20": 1729.0, + "21": 1797.0, + "22": 1780.0, + "23": 1770.0, + "24": 1801.0, + "25": 1774.0, + "26": 1874.0, + "27": 1858.0, + "28": 1739.0, + "29": 1925.0, + "30": 1854.0, + "31": 1905.0, + "32": 2023.0, + "33": 2089.0, + "34": 2000.0, + "35": 2073.0, + "36": 2038.0, + "37": 2305.0, + "38": 2124.0, + "39": 2288.0, + "40": 2376.0, + "41": 2313.0, + "42": 1979.0, + "43": 2425.0, + "44": 2321.0, + "45": 2519.0, + "46": 2453.0, + "47": 2453.0, + "48": 2709.0, + "49": 2838.0, + "50": 2677.0, + "51": 2697.0, + "52": 2743.0, + "53": 2767.0, + "54": 2883.0, + "55": 2597.0, + "56": 2784.0, + "57": 2181.0, + "58": 3618.0, + "59": 3045.0, + "60": 2919.0, + "61": 2756.0, + "62": 3075.0, + "63": 3333.0, + "64": 3814.0, + "65": 2684.0, + "66": 3152.0, + "67": 3783.0, + "68": 3437.0, + "69": 3006.0, + "70": 3342.0, + "71": 3016.0, + "72": 3033.0, + "73": 3482.0, + "74": 3295.0, + "75": 3165.0, + "76": 3366.0, + "77": 3715.0, + "78": 3210.0, + "79": 3244.0, + "80": 3165.0, + "81": 3566.0, + "82": 3059.0, + "83": 3108.0, + "84": 2783.0, + "85": 2728.0, + "86": 3321.0, + "87": 2833.0, + "88": 2913.0, + "89": 2950.0, + "90": 3842.0, + "91": 3056.0, + "92": 2852.0, + "93": 3157.0, + "94": 3313.0, + "95": 3216.0, + "96": 3504.0, + "97": 3442.0, + "98": 3310.0, + "99": 3087.0, + "100": 3154.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.68206, - "3": 0.26799, - "4": 0.2671, - "5": 0.26641, - "6": 0.31133, - "7": 0.27562, - "8": 0.26427, - "9": 0.26467, - "10": 0.26681, - "11": 0.27792, - "12": 0.26647, - "13": 0.26615, - "14": 0.26504, - "15": 0.264, - "16": 0.26697, - "17": 0.26795, - "18": 0.26802, - "19": 0.2715, - "20": 0.26665, - "21": 0.26474, - "22": 0.26577, - "23": 0.26592, - "24": 0.27183, - "25": 0.27729, - "26": 0.27451, - "27": 0.27043, - "28": 0.27215, - "29": 0.27302, - "30": 0.27219, - "31": 0.27309, - "32": 0.27237, - "33": 0.26761, - "34": 0.26582, - "35": 0.26487, - "36": 0.26491, - "37": 0.26612, - "38": 0.26506, - "39": 0.26657, - "40": 0.26566, - "41": 0.26382, - "42": 0.26404, - "43": 0.26286, - "44": 0.27058, - "45": 0.26205, - "46": 0.26569, - "47": 0.26545, - "48": 0.26636, - "49": 0.26479, - "50": 0.26649, - "51": 0.30944, - "52": 0.31633, - "53": 0.26494, - "54": 0.26768, - "55": 0.26484, - "56": 0.27388, - "57": 0.26887, - "58": 0.27093, - "59": 0.26881, - "60": 0.26864, - "61": 0.27071, - "62": 0.268, - "63": 0.2657, - "64": 0.26699, - "65": 0.26655, - "66": 0.26481, - "67": 0.26794, - "68": 0.26571, - "69": 0.26645, - "70": 0.26652, - "71": 0.26664, - "72": 0.26645, - "73": 0.26751, - "74": 0.26629, - "75": 0.26542, - "76": 0.26591, - "77": 0.26716, - "78": 0.26552, - "79": 0.27067, - "80": 0.26779, - "81": 0.2681, - "82": 0.26482, - "83": 0.26343, - "84": 0.26212, - "85": 0.26322, - "86": 0.26917, - "87": 0.26612, - "88": 0.26696, - "89": 0.26336, - "90": 0.26479, - "91": 0.26292, - "92": 0.26877, - "93": 0.28095, - "94": 0.26692, - "95": 0.26419, - "96": 0.26691, - "97": 0.26422, - "98": 0.26459, - "99": 0.2646, - "100": 0.26515 + "2": 3.9224, + "3": 0.28041, + "4": 0.27775, + "5": 0.27711, + "6": 0.27951, + "7": 0.282, + "8": 0.27696, + "9": 0.28182, + "10": 0.28282, + "11": 0.27806, + "12": 0.28294, + "13": 0.28526, + "14": 0.27851, + "15": 0.27832, + "16": 0.27993, + "17": 0.27772, + "18": 0.28003, + "19": 0.28561, + "20": 0.28045, + "21": 0.28129, + "22": 0.27937, + "23": 0.28179, + "24": 0.28076, + "25": 0.28035, + "26": 0.27754, + "27": 0.27896, + "28": 0.27823, + "29": 0.27961, + "30": 0.27717, + "31": 0.2803, + "32": 0.27841, + "33": 0.28232, + "34": 0.27983, + "35": 0.27879, + "36": 0.28021, + "37": 0.28371, + "38": 0.28298, + "39": 0.27989, + "40": 0.28041, + "41": 0.28053, + "42": 0.28174, + "43": 0.27954, + "44": 0.27803, + "45": 0.28166, + "46": 0.28244, + "47": 0.28013, + "48": 0.27906, + "49": 0.27918, + "50": 0.28122, + "51": 0.31434, + "52": 0.33352, + "53": 0.2814, + "54": 0.28294, + "55": 0.28036, + "56": 0.28571, + "57": 0.27875, + "58": 0.28236, + "59": 0.28742, + "60": 0.28046, + "61": 0.27744, + "62": 0.2834, + "63": 0.27979, + "64": 0.278, + "65": 0.2828, + "66": 0.28148, + "67": 0.27837, + "68": 0.2807, + "69": 0.90109, + "70": 0.27929, + "71": 0.27881, + "72": 0.27778, + "73": 0.27867, + "74": 0.27805, + "75": 0.27838, + "76": 0.28102, + "77": 0.28161, + "78": 0.27797, + "79": 0.28006, + "80": 0.27812, + "81": 0.28101, + "82": 0.27905, + "83": 0.28018, + "84": 0.27926, + "85": 0.278, + "86": 0.28077, + "87": 0.27888, + "88": 0.28116, + "89": 0.28208, + "90": 0.27982, + "91": 0.28039, + "92": 0.27978, + "93": 0.277, + "94": 0.27824, + "95": 0.27853, + "96": 0.27883, + "97": 0.27831, + "98": 0.27983, + "99": 0.27906, + "100": 0.2789 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json index 8d2c2b58d58..7ebc037c8d0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89463, - "2": 10.88332, - "3": 10.88064, - "4": 10.88309, - "5": 10.88332, - "6": 10.87492, - "7": 10.87689, - "8": 10.88182, - "9": 10.88055, - "10": 10.879, - "11": 10.8752, + "1": 10.89457, + "2": 10.8833, + "3": 10.88066, + "4": 10.88313, + "5": 10.88335, + "6": 10.87485, + "7": 10.87691, + "8": 10.88184, + "9": 10.88054, + "10": 10.87894, + "11": 10.87523, "12": 10.87014, - "13": 10.86618, - "14": 10.86433, - "15": 10.83999, - "16": 10.83442, - "17": 10.84483, - "18": 10.81981, - "19": 10.84023, - "20": 10.75974, - "21": 10.74842, - "22": 10.73585, - "23": 10.72846, - "24": 10.70204, - "25": 10.69386, - "26": 10.68294, - "27": 10.65042, - "28": 10.58861, - "29": 10.55276, - "30": 10.52991, - "31": 10.52067, - "32": 10.51487, - "33": 10.47672, - "34": 10.45098, - "35": 10.45204, - "36": 10.42467, - "37": 10.3935, - "38": 10.39796, - "39": 10.36701, - "40": 10.35717, - "41": 10.33082, - "42": 10.31015, - "43": 10.28972, - "44": 10.26722, - "45": 10.27276, - "46": 10.23956, - "47": 10.22509, + "13": 10.86616, + "14": 10.86431, + "15": 10.84011, + "16": 10.83443, + "17": 10.84496, + "18": 10.81997, + "19": 10.84015, + "20": 10.75975, + "21": 10.74843, + "22": 10.73591, + "23": 10.72848, + "24": 10.70201, + "25": 10.69395, + "26": 10.68291, + "27": 10.65044, + "28": 10.58865, + "29": 10.5528, + "30": 10.53001, + "31": 10.52071, + "32": 10.51493, + "33": 10.47669, + "34": 10.45106, + "35": 10.45202, + "36": 10.42475, + "37": 10.39359, + "38": 10.39804, + "39": 10.36707, + "40": 10.35722, + "41": 10.33091, + "42": 10.31022, + "43": 10.28973, + "44": 10.26728, + "45": 10.27285, + "46": 10.23962, + "47": 10.22516, "48": 10.18113, - "49": 10.18221, - "50": 10.18109, - "51": 10.18054, - "52": 10.13848, - "53": 10.14381, - "54": 10.10815, - "55": 10.08196, - "56": 10.10878, - "57": 10.09685, + "49": 10.18226, + "50": 10.1811, + "51": 10.18062, + "52": 10.13852, + "53": 10.14386, + "54": 10.10818, + "55": 10.08201, + "56": 10.10885, + "57": 10.09687, "58": 10.11263, - "59": 10.06067, + "59": 10.06068, "60": 10.08081, - "61": 10.029, - "62": 10.00208, - "63": 10.07351, - "64": 10.02973, - "65": 10.00767, - "66": 10.02968, - "67": 10.00724, - "68": 9.96811, - "69": 9.99247, - "70": 9.96962, - "71": 9.99959, - "72": 9.97932, + "61": 10.02902, + "62": 10.00206, + "63": 10.07353, + "64": 10.02981, + "65": 10.0077, + "66": 10.02965, + "67": 10.00722, + "68": 9.96815, + "69": 9.99249, + "70": 9.96967, + "71": 9.99962, + "72": 9.97935, "73": 9.97019, - "74": 9.9547, - "75": 9.93058, - "76": 9.96382, - "77": 9.95341, - "78": 9.90812, - "79": 9.91503, - "80": 9.92521, - "81": 9.95289, - "82": 9.88577, - "83": 9.85328, - "84": 9.79572, - "85": 9.78589, - "86": 9.87801, - "87": 9.90219, - "88": 9.87507, - "89": 9.82262, - "90": 9.81115, - "91": 9.82642, - "92": 9.81572, - "93": 9.75371, - "94": 9.82278, - "95": 9.81996, - "96": 9.80283, - "97": 9.74847, - "98": 9.77372, - "99": 9.81917, - "100": 9.71201 + "74": 9.95472, + "75": 9.93064, + "76": 9.96386, + "77": 9.95355, + "78": 9.90822, + "79": 9.91513, + "80": 9.92528, + "81": 9.95294, + "82": 9.88587, + "83": 9.85333, + "84": 9.79576, + "85": 9.78595, + "86": 9.87805, + "87": 9.90221, + "88": 9.87509, + "89": 9.82267, + "90": 9.81121, + "91": 9.82646, + "92": 9.81581, + "93": 9.75374, + "94": 9.82283, + "95": 9.81998, + "96": 9.80288, + "97": 9.74858, + "98": 9.77376, + "99": 9.81918, + "100": 9.71205 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1922.0, - "2": 1807.0, - "3": 1882.0, - "4": 1775.0, - "5": 1912.0, - "6": 1714.0, - "7": 1974.0, - "8": 1935.0, - "9": 1833.0, - "10": 1932.0, - "11": 1923.0, - "12": 1769.0, - "13": 1796.0, - "14": 2001.0, - "15": 1767.0, - "16": 1851.0, - "17": 1834.0, - "18": 1796.0, - "19": 1749.0, - "20": 1655.0, - "21": 1824.0, - "22": 1766.0, - "23": 1820.0, - "24": 1931.0, - "25": 1746.0, - "26": 1895.0, - "27": 1859.0, - "28": 1860.0, - "29": 1939.0, - "30": 1870.0, - "31": 2013.0, - "32": 1982.0, - "33": 2040.0, - "34": 2102.0, + "1": 1834.0, + "2": 1741.0, + "3": 1887.0, + "4": 1793.0, + "5": 1911.0, + "6": 1830.0, + "7": 1956.0, + "8": 1826.0, + "9": 1873.0, + "10": 1824.0, + "11": 1830.0, + "12": 1861.0, + "13": 1810.0, + "14": 1928.0, + "15": 1787.0, + "16": 1797.0, + "17": 1926.0, + "18": 1868.0, + "19": 1734.0, + "20": 1707.0, + "21": 2002.0, + "22": 1769.0, + "23": 1784.0, + "24": 1993.0, + "25": 1663.0, + "26": 1799.0, + "27": 1819.0, + "28": 1793.0, + "29": 1978.0, + "30": 1846.0, + "31": 2065.0, + "32": 1998.0, + "33": 1969.0, + "34": 2030.0, "35": 2075.0, - "36": 2026.0, - "37": 2314.0, - "38": 2141.0, - "39": 2189.0, - "40": 2255.0, - "41": 2396.0, - "42": 1953.0, - "43": 2389.0, - "44": 2234.0, - "45": 2588.0, - "46": 2583.0, - "47": 2581.0, - "48": 2757.0, - "49": 2847.0, - "50": 2589.0, - "51": 2438.0, - "52": 2832.0, - "53": 2666.0, - "54": 2900.0, - "55": 2729.0, - "56": 2824.0, - "57": 2182.0, - "58": 3842.0, - "59": 2971.0, - "60": 3042.0, - "61": 2689.0, - "62": 3269.0, - "63": 3494.0, - "64": 3553.0, - "65": 2767.0, - "66": 3146.0, - "67": 3952.0, - "68": 3552.0, - "69": 3054.0, - "70": 3377.0, - "71": 3173.0, - "72": 2909.0, - "73": 3460.0, - "74": 3310.0, - "75": 3240.0, - "76": 3197.0, - "77": 3644.0, - "78": 3322.0, - "79": 3270.0, - "80": 3197.0, - "81": 3406.0, - "82": 2921.0, - "83": 2929.0, - "84": 2873.0, - "85": 2722.0, - "86": 3294.0, - "87": 2854.0, - "88": 3109.0, - "89": 3067.0, - "90": 3859.0, - "91": 2975.0, - "92": 3083.0, - "93": 3074.0, - "94": 3298.0, - "95": 3271.0, - "96": 3531.0, - "97": 3566.0, - "98": 3646.0, - "99": 3246.0, - "100": 3560.0 + "36": 2019.0, + "37": 2275.0, + "38": 2039.0, + "39": 2182.0, + "40": 2303.0, + "41": 2367.0, + "42": 2060.0, + "43": 2514.0, + "44": 2224.0, + "45": 2665.0, + "46": 2587.0, + "47": 2608.0, + "48": 2761.0, + "49": 2875.0, + "50": 2566.0, + "51": 2495.0, + "52": 2745.0, + "53": 2716.0, + "54": 2941.0, + "55": 2646.0, + "56": 2783.0, + "57": 2219.0, + "58": 3749.0, + "59": 3080.0, + "60": 2948.0, + "61": 2729.0, + "62": 3297.0, + "63": 3479.0, + "64": 3568.0, + "65": 2790.0, + "66": 3159.0, + "67": 3953.0, + "68": 3534.0, + "69": 3111.0, + "70": 3394.0, + "71": 3076.0, + "72": 3103.0, + "73": 3502.0, + "74": 3458.0, + "75": 3260.0, + "76": 3172.0, + "77": 3647.0, + "78": 3314.0, + "79": 3252.0, + "80": 3179.0, + "81": 3383.0, + "82": 2774.0, + "83": 3012.0, + "84": 2861.0, + "85": 2746.0, + "86": 3245.0, + "87": 2921.0, + "88": 3137.0, + "89": 3101.0, + "90": 3855.0, + "91": 3068.0, + "92": 2903.0, + "93": 3124.0, + "94": 3341.0, + "95": 3345.0, + "96": 3457.0, + "97": 3584.0, + "98": 3644.0, + "99": 3274.0, + "100": 3526.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.5611, - "3": 3.12672, - "4": 2.50292, - "5": 2.11785, - "6": 2.60046, - "7": 2.21771, - "8": 3.05636, - "9": 3.1505, - "10": 2.70528, - "11": 3.00452, - "12": 2.61449, - "13": 2.82307, - "14": 3.18549, - "15": 2.47155, - "16": 2.67596, - "17": 3.55548, - "18": 2.7566, - "19": 2.92395, - "20": 2.62166, - "21": 2.92505, - "22": 2.36191, - "23": 2.89319, - "24": 2.37961, - "25": 3.02937, - "26": 3.18521, - "27": 2.63733, - "28": 3.15102, - "29": 3.36992, - "30": 3.93325, - "31": 3.37954, - "32": 6.69137, - "33": 3.03178, - "34": 2.9217, - "35": 3.51875, - "36": 3.67603, - "37": 2.90199, - "38": 2.57109, - "39": 4.10516, - "40": 3.70257, - "41": 3.95061, - "42": 3.62745, - "43": 2.46073, - "44": 2.61556, - "45": 3.35226, - "46": 2.33188, - "47": 2.38682, - "48": 3.00986, - "49": 2.16372, - "50": 2.58813, - "51": 2.2218, - "52": 3.04153, - "53": 2.41338, - "54": 2.93755, - "55": 2.45756, - "56": 3.56881, - "57": 2.0903, - "58": 3.0333, - "59": 2.50846, - "60": 2.4666, - "61": 3.10107, - "62": 3.03738, - "63": 2.75714, - "64": 2.71472, - "65": 2.50203, - "66": 2.69262, - "67": 2.78985, - "68": 1.8342, - "69": 2.59825, - "70": 2.26667, - "71": 2.84191, - "72": 2.27394, - "73": 2.40866, - "74": 2.20699, - "75": 2.31441, - "76": 2.50752, - "77": 2.35908, - "78": 2.4069, - "79": 3.1483, - "80": 2.8069, - "81": 3.01487, - "82": 2.9794, - "83": 3.03638, - "84": 2.45697, - "85": 3.09786, - "86": 2.35509, - "87": 2.9868, - "88": 2.69431, - "89": 2.69174, - "90": 2.54919, - "91": 2.48305, - "92": 2.8429, - "93": 2.0244, - "94": 2.53131, - "95": 2.49735, - "96": 2.50347, - "97": 2.23901, - "98": 2.39175, - "99": 2.89921, - "100": 2.06984 + "2": 5.24611, + "3": 2.67805, + "4": 2.44902, + "5": 1.98407, + "6": 2.52332, + "7": 2.04329, + "8": 2.82692, + "9": 2.92773, + "10": 2.37962, + "11": 2.59081, + "12": 2.06909, + "13": 2.45433, + "14": 2.64738, + "15": 2.1176, + "16": 2.31158, + "17": 2.99012, + "18": 2.35015, + "19": 2.47711, + "20": 2.2539, + "21": 2.55841, + "22": 2.18678, + "23": 2.64441, + "24": 2.11538, + "25": 2.9649, + "26": 3.21403, + "27": 2.56323, + "28": 2.34983, + "29": 2.20187, + "30": 2.37981, + "31": 2.25146, + "32": 3.6027, + "33": 1.99066, + "34": 2.0238, + "35": 2.97185, + "36": 3.01659, + "37": 2.34621, + "38": 1.78694, + "39": 2.55787, + "40": 2.14748, + "41": 2.22769, + "42": 2.80579, + "43": 1.99045, + "44": 2.03504, + "45": 2.53392, + "46": 1.95135, + "47": 2.07268, + "48": 2.33796, + "49": 1.87359, + "50": 2.33397, + "51": 1.36776, + "52": 2.43164, + "53": 2.28629, + "54": 2.37446, + "55": 2.00903, + "56": 2.7094, + "57": 1.50248, + "58": 2.03607, + "59": 1.83749, + "60": 1.94497, + "61": 2.35343, + "62": 2.57454, + "63": 1.79527, + "64": 2.52763, + "65": 2.16905, + "66": 2.29036, + "67": 2.63455, + "68": 1.72012, + "69": 2.44255, + "70": 2.25223, + "71": 2.86426, + "72": 2.18352, + "73": 2.31721, + "74": 2.12295, + "75": 2.10264, + "76": 2.36883, + "77": 2.24576, + "78": 2.27814, + "79": 2.75811, + "80": 2.28109, + "81": 2.71544, + "82": 2.56674, + "83": 2.40671, + "84": 1.9543, + "85": 2.11635, + "86": 2.14639, + "87": 2.73001, + "88": 2.45847, + "89": 2.5278, + "90": 2.56311, + "91": 2.32026, + "92": 2.81907, + "93": 2.07457, + "94": 2.44409, + "95": 2.30936, + "96": 2.34601, + "97": 2.04897, + "98": 2.21601, + "99": 2.8261, + "100": 2.09876 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json index 30cbdd56c51..57e582d7bf7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89723, - "2": 10.89039, - "3": 10.88307, - "4": 10.88936, - "5": 10.88232, - "6": 10.88662, - "7": 10.88761, - "8": 10.88217, - "9": 10.8849, - "10": 10.88111, - "11": 10.87931, - "12": 10.87751, - "13": 10.87274, - "14": 10.8683, - "15": 10.84624, - "16": 10.83789, - "17": 10.83723, - "18": 10.82685, - "19": 10.83392, - "20": 10.76073, - "21": 10.74166, - "22": 10.72525, - "23": 10.71959, - "24": 10.70182, - "25": 10.69462, - "26": 10.67756, - "27": 10.65206, - "28": 10.57443, - "29": 10.5504, - "30": 10.52251, - "31": 10.5259, - "32": 10.51363, - "33": 10.46608, - "34": 10.4379, - "35": 10.43775, - "36": 10.42159, - "37": 10.38933, - "38": 10.38943, - "39": 10.3568, - "40": 10.34751, - "41": 10.32304, - "42": 10.30556, - "43": 10.28355, + "1": 10.89725, + "2": 10.89034, + "3": 10.88313, + "4": 10.88928, + "5": 10.8823, + "6": 10.88649, + "7": 10.88759, + "8": 10.88216, + "9": 10.88485, + "10": 10.88115, + "11": 10.87936, + "12": 10.87759, + "13": 10.87278, + "14": 10.86819, + "15": 10.84637, + "16": 10.83795, + "17": 10.8373, + "18": 10.82682, + "19": 10.83389, + "20": 10.76081, + "21": 10.74174, + "22": 10.72524, + "23": 10.71965, + "24": 10.70188, + "25": 10.69461, + "26": 10.67754, + "27": 10.6521, + "28": 10.57447, + "29": 10.55048, + "30": 10.52256, + "31": 10.52596, + "32": 10.51369, + "33": 10.46614, + "34": 10.43797, + "35": 10.43785, + "36": 10.42172, + "37": 10.38938, + "38": 10.38955, + "39": 10.35676, + "40": 10.34758, + "41": 10.32306, + "42": 10.30569, + "43": 10.28356, "44": 10.26322, - "45": 10.2662, - "46": 10.2368, - "47": 10.22146, - "48": 10.17272, - "49": 10.18156, - "50": 10.17791, - "51": 10.18023, - "52": 10.13377, - "53": 10.13153, - "54": 10.10638, - "55": 10.08296, - "56": 10.10637, - "57": 10.10201, + "45": 10.26623, + "46": 10.23684, + "47": 10.22155, + "48": 10.17277, + "49": 10.18161, + "50": 10.17798, + "51": 10.18021, + "52": 10.13376, + "53": 10.13158, + "54": 10.10647, + "55": 10.08302, + "56": 10.10644, + "57": 10.10207, "58": 10.10817, - "59": 10.04452, - "60": 10.07106, - "61": 10.02559, - "62": 9.9984, - "63": 10.06817, - "64": 10.01965, - "65": 9.98079, - "66": 10.02431, - "67": 10.00269, - "68": 9.96782, - "69": 9.98685, - "70": 9.96672, - "71": 9.99227, - "72": 9.97696, + "59": 10.04457, + "60": 10.07112, + "61": 10.0256, + "62": 9.99842, + "63": 10.06818, + "64": 10.01971, + "65": 9.98073, + "66": 10.02437, + "67": 10.0027, + "68": 9.96789, + "69": 9.98687, + "70": 9.96676, + "71": 9.9923, + "72": 9.97702, "73": 9.96099, - "74": 9.95636, - "75": 9.92492, - "76": 9.95108, - "77": 9.94711, - "78": 9.9007, - "79": 9.90293, - "80": 9.9174, - "81": 9.94305, - "82": 9.88166, - "83": 9.84046, - "84": 9.78038, - "85": 9.77116, - "86": 9.87352, - "87": 9.90184, - "88": 9.87564, - "89": 9.82672, - "90": 9.81115, - "91": 9.82071, - "92": 9.81382, - "93": 9.7493, - "94": 9.81784, - "95": 9.80819, - "96": 9.7946, - "97": 9.74391, - "98": 9.76747, - "99": 9.81868, - "100": 9.705 + "74": 9.95639, + "75": 9.92493, + "76": 9.95112, + "77": 9.94724, + "78": 9.90078, + "79": 9.90296, + "80": 9.91747, + "81": 9.94309, + "82": 9.88173, + "83": 9.84051, + "84": 9.78052, + "85": 9.7712, + "86": 9.87355, + "87": 9.9019, + "88": 9.87567, + "89": 9.82676, + "90": 9.81117, + "91": 9.82076, + "92": 9.81381, + "93": 9.74936, + "94": 9.81789, + "95": 9.80823, + "96": 9.79458, + "97": 9.74393, + "98": 9.76746, + "99": 9.81869, + "100": 9.70503 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1842.0, - "2": 1747.0, - "3": 1798.0, - "4": 1756.0, - "5": 1843.0, - "6": 1866.0, - "7": 1878.0, - "8": 1740.0, - "9": 1664.0, - "10": 1710.0, - "11": 1759.0, - "12": 1778.0, - "13": 1753.0, - "14": 1927.0, - "15": 1734.0, - "16": 1783.0, - "17": 1838.0, - "18": 1749.0, - "19": 1687.0, - "20": 1660.0, - "21": 1807.0, - "22": 1826.0, - "23": 1718.0, - "24": 1932.0, - "25": 1716.0, - "26": 1844.0, - "27": 1871.0, - "28": 1786.0, - "29": 1949.0, - "30": 1805.0, - "31": 2018.0, - "32": 2076.0, - "33": 1949.0, - "34": 2154.0, - "35": 2209.0, - "36": 2126.0, - "37": 2259.0, - "38": 2249.0, - "39": 2310.0, - "40": 2346.0, - "41": 2381.0, - "42": 2041.0, - "43": 2284.0, - "44": 2233.0, - "45": 2672.0, - "46": 2483.0, - "47": 2560.0, - "48": 2663.0, - "49": 2907.0, - "50": 2702.0, - "51": 2562.0, - "52": 2823.0, - "53": 2633.0, - "54": 2876.0, - "55": 2635.0, - "56": 2793.0, - "57": 2287.0, - "58": 3592.0, - "59": 3073.0, - "60": 2865.0, - "61": 2722.0, - "62": 3098.0, - "63": 3278.0, - "64": 3668.0, - "65": 2763.0, - "66": 3267.0, - "67": 3872.0, - "68": 3554.0, - "69": 3020.0, - "70": 3285.0, - "71": 3065.0, - "72": 3037.0, - "73": 3495.0, - "74": 3198.0, - "75": 3274.0, - "76": 3402.0, - "77": 3744.0, - "78": 3175.0, - "79": 3312.0, - "80": 3230.0, - "81": 3448.0, - "82": 2991.0, - "83": 3114.0, - "84": 2796.0, - "85": 2725.0, - "86": 3304.0, - "87": 2802.0, - "88": 3066.0, - "89": 2962.0, - "90": 3757.0, - "91": 3102.0, - "92": 2938.0, - "93": 3086.0, - "94": 3253.0, - "95": 3251.0, - "96": 3529.0, - "97": 3575.0, - "98": 3498.0, - "99": 3111.0, - "100": 3147.0 + "1": 1856.0, + "2": 1781.0, + "3": 1802.0, + "4": 1768.0, + "5": 1858.0, + "6": 1894.0, + "7": 1967.0, + "8": 1731.0, + "9": 1828.0, + "10": 1828.0, + "11": 1905.0, + "12": 1779.0, + "13": 1772.0, + "14": 1893.0, + "15": 1713.0, + "16": 1852.0, + "17": 1776.0, + "18": 1843.0, + "19": 1633.0, + "20": 1729.0, + "21": 1797.0, + "22": 1780.0, + "23": 1770.0, + "24": 1801.0, + "25": 1774.0, + "26": 1874.0, + "27": 1858.0, + "28": 1739.0, + "29": 1925.0, + "30": 1854.0, + "31": 1905.0, + "32": 2023.0, + "33": 2089.0, + "34": 2000.0, + "35": 2073.0, + "36": 2038.0, + "37": 2305.0, + "38": 2124.0, + "39": 2288.0, + "40": 2376.0, + "41": 2313.0, + "42": 1979.0, + "43": 2425.0, + "44": 2321.0, + "45": 2519.0, + "46": 2453.0, + "47": 2453.0, + "48": 2709.0, + "49": 2838.0, + "50": 2677.0, + "51": 2697.0, + "52": 2743.0, + "53": 2767.0, + "54": 2883.0, + "55": 2597.0, + "56": 2784.0, + "57": 2181.0, + "58": 3618.0, + "59": 3045.0, + "60": 2919.0, + "61": 2756.0, + "62": 3075.0, + "63": 3333.0, + "64": 3814.0, + "65": 2684.0, + "66": 3152.0, + "67": 3783.0, + "68": 3437.0, + "69": 3006.0, + "70": 3342.0, + "71": 3016.0, + "72": 3033.0, + "73": 3482.0, + "74": 3295.0, + "75": 3165.0, + "76": 3366.0, + "77": 3715.0, + "78": 3210.0, + "79": 3244.0, + "80": 3165.0, + "81": 3566.0, + "82": 3059.0, + "83": 3108.0, + "84": 2783.0, + "85": 2728.0, + "86": 3321.0, + "87": 2833.0, + "88": 2913.0, + "89": 2950.0, + "90": 3842.0, + "91": 3056.0, + "92": 2852.0, + "93": 3157.0, + "94": 3313.0, + "95": 3216.0, + "96": 3504.0, + "97": 3442.0, + "98": 3310.0, + "99": 3087.0, + "100": 3154.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.09229, - "3": 0.28973, - "4": 0.28117, - "5": 0.28002, - "6": 0.28054, - "7": 0.28075, - "8": 0.30829, - "9": 0.29364, - "10": 0.29581, - "11": 0.30981, - "12": 0.30276, - "13": 0.28494, - "14": 0.28073, - "15": 0.28508, - "16": 0.28287, - "17": 0.28038, - "18": 0.28078, - "19": 0.28009, - "20": 0.28575, - "21": 0.2791, - "22": 0.28316, - "23": 0.28062, - "24": 0.2796, - "25": 0.27874, - "26": 0.28657, - "27": 0.2803, - "28": 0.28039, - "29": 0.28157, - "30": 0.2804, - "31": 0.27957, - "32": 0.2786, - "33": 0.27909, - "34": 0.27978, - "35": 0.28196, - "36": 0.28427, - "37": 0.27865, - "38": 0.27838, - "39": 0.28176, - "40": 0.28164, - "41": 0.28141, - "42": 0.27817, - "43": 0.28011, - "44": 0.28334, - "45": 0.27989, - "46": 0.28069, - "47": 0.27997, - "48": 0.2789, - "49": 0.27814, - "50": 0.28065, - "51": 0.31687, - "52": 0.33462, - "53": 0.28535, - "54": 0.28977, - "55": 0.28138, - "56": 0.2793, - "57": 0.28052, - "58": 0.28126, - "59": 0.2805, - "60": 0.28272, - "61": 0.28265, - "62": 0.28024, - "63": 0.28263, - "64": 0.27999, - "65": 0.28232, - "66": 0.28135, - "67": 0.28159, - "68": 0.28314, - "69": 0.28021, - "70": 0.28044, - "71": 0.28249, - "72": 0.28144, - "73": 0.28031, - "74": 0.28248, - "75": 0.28182, - "76": 0.28026, - "77": 0.27954, - "78": 0.27843, - "79": 0.28015, - "80": 0.27921, - "81": 0.27873, - "82": 0.27856, - "83": 0.27898, - "84": 0.27835, - "85": 0.27764, - "86": 0.27869, - "87": 0.2796, - "88": 0.28004, - "89": 0.28243, - "90": 0.27895, - "91": 0.28368, - "92": 0.27711, - "93": 0.27854, - "94": 0.28103, - "95": 0.27794, - "96": 0.27842, - "97": 0.2811, - "98": 0.28651, - "99": 0.27871, - "100": 0.28202 + "2": 4.102, + "3": 0.29192, + "4": 0.29186, + "5": 0.29501, + "6": 0.28747, + "7": 0.28438, + "8": 0.28958, + "9": 0.28606, + "10": 0.28695, + "11": 0.29327, + "12": 0.28649, + "13": 0.2869, + "14": 0.28065, + "15": 0.28602, + "16": 0.28368, + "17": 0.28587, + "18": 0.28364, + "19": 0.28977, + "20": 0.2917, + "21": 0.29301, + "22": 0.2958, + "23": 0.29553, + "24": 0.29517, + "25": 0.29461, + "26": 0.29613, + "27": 0.29442, + "28": 0.29491, + "29": 0.29098, + "30": 0.29462, + "31": 0.29557, + "32": 0.29538, + "33": 0.29648, + "34": 0.2935, + "35": 0.29344, + "36": 0.29934, + "37": 0.29497, + "38": 0.29426, + "39": 0.29939, + "40": 0.29345, + "41": 0.2925, + "42": 0.29355, + "43": 0.29996, + "44": 0.292, + "45": 0.29315, + "46": 0.2932, + "47": 0.295, + "48": 0.29177, + "49": 0.29341, + "50": 0.29145, + "51": 0.33447, + "52": 0.35798, + "53": 0.29807, + "54": 0.29328, + "55": 0.29292, + "56": 0.29206, + "57": 0.29339, + "58": 0.29038, + "59": 0.29256, + "60": 0.29557, + "61": 0.29416, + "62": 0.29285, + "63": 0.29124, + "64": 0.2905, + "65": 0.29014, + "66": 0.29456, + "67": 0.29313, + "68": 0.29189, + "69": 0.29143, + "70": 0.29274, + "71": 0.29047, + "72": 0.29099, + "73": 0.28974, + "74": 0.28966, + "75": 0.29148, + "76": 0.28992, + "77": 0.28956, + "78": 0.29196, + "79": 0.29504, + "80": 0.29297, + "81": 0.29928, + "82": 0.2902, + "83": 0.28916, + "84": 0.29195, + "85": 0.28858, + "86": 0.28915, + "87": 0.28923, + "88": 0.29339, + "89": 0.28936, + "90": 0.2944, + "91": 0.28838, + "92": 0.29164, + "93": 0.29013, + "94": 0.29184, + "95": 0.28872, + "96": 0.28854, + "97": 0.29199, + "98": 0.29086, + "99": 0.29106, + "100": 0.29571 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json index 0bd2ea0961b..a03b5dff6fa 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89463, - "2": 10.88332, - "3": 10.88064, - "4": 10.88309, - "5": 10.88332, - "6": 10.87492, - "7": 10.87689, - "8": 10.88182, - "9": 10.88055, - "10": 10.879, - "11": 10.8752, + "1": 10.89457, + "2": 10.8833, + "3": 10.88066, + "4": 10.88313, + "5": 10.88335, + "6": 10.87485, + "7": 10.87691, + "8": 10.88184, + "9": 10.88054, + "10": 10.87894, + "11": 10.87523, "12": 10.87014, - "13": 10.86618, - "14": 10.86433, - "15": 10.83999, - "16": 10.83442, - "17": 10.84483, - "18": 10.81981, - "19": 10.84023, - "20": 10.75974, - "21": 10.74842, - "22": 10.73585, - "23": 10.72846, - "24": 10.70204, - "25": 10.69386, - "26": 10.68294, - "27": 10.65042, - "28": 10.58861, - "29": 10.55276, - "30": 10.52991, - "31": 10.52067, - "32": 10.51487, - "33": 10.47672, - "34": 10.45098, - "35": 10.45204, - "36": 10.42467, - "37": 10.3935, - "38": 10.39796, - "39": 10.36701, - "40": 10.35717, - "41": 10.33082, - "42": 10.31015, - "43": 10.28972, - "44": 10.26722, - "45": 10.27276, - "46": 10.23956, - "47": 10.22509, + "13": 10.86616, + "14": 10.86431, + "15": 10.84011, + "16": 10.83443, + "17": 10.84496, + "18": 10.81997, + "19": 10.84015, + "20": 10.75975, + "21": 10.74843, + "22": 10.73591, + "23": 10.72848, + "24": 10.70201, + "25": 10.69395, + "26": 10.68291, + "27": 10.65044, + "28": 10.58865, + "29": 10.5528, + "30": 10.53001, + "31": 10.52071, + "32": 10.51493, + "33": 10.47669, + "34": 10.45106, + "35": 10.45202, + "36": 10.42475, + "37": 10.39359, + "38": 10.39804, + "39": 10.36707, + "40": 10.35722, + "41": 10.33091, + "42": 10.31022, + "43": 10.28973, + "44": 10.26728, + "45": 10.27285, + "46": 10.23962, + "47": 10.22516, "48": 10.18113, - "49": 10.18221, - "50": 10.18109, - "51": 10.18054, - "52": 10.13848, - "53": 10.14381, - "54": 10.10815, - "55": 10.08196, - "56": 10.10878, - "57": 10.09685, + "49": 10.18226, + "50": 10.1811, + "51": 10.18062, + "52": 10.13852, + "53": 10.14386, + "54": 10.10818, + "55": 10.08201, + "56": 10.10885, + "57": 10.09687, "58": 10.11263, - "59": 10.06067, + "59": 10.06068, "60": 10.08081, - "61": 10.029, - "62": 10.00208, - "63": 10.07351, - "64": 10.02973, - "65": 10.00767, - "66": 10.02968, - "67": 10.00724, - "68": 9.96811, - "69": 9.99247, - "70": 9.96962, - "71": 9.99959, - "72": 9.97932, + "61": 10.02902, + "62": 10.00206, + "63": 10.07353, + "64": 10.02981, + "65": 10.0077, + "66": 10.02965, + "67": 10.00722, + "68": 9.96815, + "69": 9.99249, + "70": 9.96967, + "71": 9.99962, + "72": 9.97935, "73": 9.97019, - "74": 9.9547, - "75": 9.93058, - "76": 9.96382, - "77": 9.95341, - "78": 9.90812, - "79": 9.91503, - "80": 9.92521, - "81": 9.95289, - "82": 9.88577, - "83": 9.85328, - "84": 9.79572, - "85": 9.78589, - "86": 9.87801, - "87": 9.90219, - "88": 9.87507, - "89": 9.82262, - "90": 9.81115, - "91": 9.82642, - "92": 9.81572, - "93": 9.75371, - "94": 9.82278, - "95": 9.81996, - "96": 9.80283, - "97": 9.74847, - "98": 9.77372, - "99": 9.81917, - "100": 9.71201 + "74": 9.95472, + "75": 9.93064, + "76": 9.96386, + "77": 9.95355, + "78": 9.90822, + "79": 9.91513, + "80": 9.92528, + "81": 9.95294, + "82": 9.88587, + "83": 9.85333, + "84": 9.79576, + "85": 9.78595, + "86": 9.87805, + "87": 9.90221, + "88": 9.87509, + "89": 9.82267, + "90": 9.81121, + "91": 9.82646, + "92": 9.81581, + "93": 9.75374, + "94": 9.82283, + "95": 9.81998, + "96": 9.80288, + "97": 9.74858, + "98": 9.77376, + "99": 9.81918, + "100": 9.71205 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1922.0, - "2": 1807.0, - "3": 1882.0, - "4": 1775.0, - "5": 1912.0, - "6": 1714.0, - "7": 1974.0, - "8": 1935.0, - "9": 1833.0, - "10": 1932.0, - "11": 1923.0, - "12": 1769.0, - "13": 1796.0, - "14": 2001.0, - "15": 1767.0, - "16": 1851.0, - "17": 1834.0, - "18": 1796.0, - "19": 1749.0, - "20": 1655.0, - "21": 1824.0, - "22": 1766.0, - "23": 1820.0, - "24": 1931.0, - "25": 1746.0, - "26": 1895.0, - "27": 1859.0, - "28": 1860.0, - "29": 1939.0, - "30": 1870.0, - "31": 2013.0, - "32": 1982.0, - "33": 2040.0, - "34": 2102.0, + "1": 1834.0, + "2": 1741.0, + "3": 1887.0, + "4": 1793.0, + "5": 1911.0, + "6": 1830.0, + "7": 1956.0, + "8": 1826.0, + "9": 1873.0, + "10": 1824.0, + "11": 1830.0, + "12": 1861.0, + "13": 1810.0, + "14": 1928.0, + "15": 1787.0, + "16": 1797.0, + "17": 1926.0, + "18": 1868.0, + "19": 1734.0, + "20": 1707.0, + "21": 2002.0, + "22": 1769.0, + "23": 1784.0, + "24": 1993.0, + "25": 1663.0, + "26": 1799.0, + "27": 1819.0, + "28": 1793.0, + "29": 1978.0, + "30": 1846.0, + "31": 2065.0, + "32": 1998.0, + "33": 1969.0, + "34": 2030.0, "35": 2075.0, - "36": 2026.0, - "37": 2314.0, - "38": 2141.0, - "39": 2189.0, - "40": 2255.0, - "41": 2396.0, - "42": 1953.0, - "43": 2389.0, - "44": 2234.0, - "45": 2588.0, - "46": 2583.0, - "47": 2581.0, - "48": 2757.0, - "49": 2847.0, - "50": 2589.0, - "51": 2438.0, - "52": 2832.0, - "53": 2666.0, - "54": 2900.0, - "55": 2729.0, - "56": 2824.0, - "57": 2182.0, - "58": 3842.0, - "59": 2971.0, - "60": 3042.0, - "61": 2689.0, - "62": 3269.0, - "63": 3494.0, - "64": 3553.0, - "65": 2767.0, - "66": 3146.0, - "67": 3952.0, - "68": 3552.0, - "69": 3054.0, - "70": 3377.0, - "71": 3173.0, - "72": 2909.0, - "73": 3460.0, - "74": 3310.0, - "75": 3240.0, - "76": 3197.0, - "77": 3644.0, - "78": 3322.0, - "79": 3270.0, - "80": 3197.0, - "81": 3406.0, - "82": 2921.0, - "83": 2929.0, - "84": 2873.0, - "85": 2722.0, - "86": 3294.0, - "87": 2854.0, - "88": 3109.0, - "89": 3067.0, - "90": 3859.0, - "91": 2975.0, - "92": 3083.0, - "93": 3074.0, - "94": 3298.0, - "95": 3271.0, - "96": 3531.0, - "97": 3566.0, - "98": 3646.0, - "99": 3246.0, - "100": 3560.0 + "36": 2019.0, + "37": 2275.0, + "38": 2039.0, + "39": 2182.0, + "40": 2303.0, + "41": 2367.0, + "42": 2060.0, + "43": 2514.0, + "44": 2224.0, + "45": 2665.0, + "46": 2587.0, + "47": 2608.0, + "48": 2761.0, + "49": 2875.0, + "50": 2566.0, + "51": 2495.0, + "52": 2745.0, + "53": 2716.0, + "54": 2941.0, + "55": 2646.0, + "56": 2783.0, + "57": 2219.0, + "58": 3749.0, + "59": 3080.0, + "60": 2948.0, + "61": 2729.0, + "62": 3297.0, + "63": 3479.0, + "64": 3568.0, + "65": 2790.0, + "66": 3159.0, + "67": 3953.0, + "68": 3534.0, + "69": 3111.0, + "70": 3394.0, + "71": 3076.0, + "72": 3103.0, + "73": 3502.0, + "74": 3458.0, + "75": 3260.0, + "76": 3172.0, + "77": 3647.0, + "78": 3314.0, + "79": 3252.0, + "80": 3179.0, + "81": 3383.0, + "82": 2774.0, + "83": 3012.0, + "84": 2861.0, + "85": 2746.0, + "86": 3245.0, + "87": 2921.0, + "88": 3137.0, + "89": 3101.0, + "90": 3855.0, + "91": 3068.0, + "92": 2903.0, + "93": 3124.0, + "94": 3341.0, + "95": 3345.0, + "96": 3457.0, + "97": 3584.0, + "98": 3644.0, + "99": 3274.0, + "100": 3526.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21012, - "3": 2.48683, - "4": 2.08758, - "5": 1.84106, - "6": 2.36818, - "7": 1.78167, - "8": 2.56082, - "9": 2.63238, - "10": 2.116, - "11": 2.29128, - "12": 1.9369, - "13": 2.21414, - "14": 2.3711, - "15": 1.78324, - "16": 2.07424, - "17": 2.64812, - "18": 1.913, - "19": 2.2414, - "20": 1.9884, - "21": 2.29313, - "22": 1.78308, - "23": 2.31951, - "24": 1.83828, - "25": 2.42959, - "26": 2.57941, - "27": 1.94076, - "28": 1.96561, - "29": 1.76661, - "30": 1.97064, - "31": 1.77809, - "32": 2.94694, - "33": 1.78611, - "34": 1.71801, - "35": 2.43344, - "36": 2.30883, - "37": 2.07707, - "38": 1.58036, - "39": 2.52887, - "40": 2.02749, - "41": 2.17248, - "42": 2.71556, - "43": 1.90625, - "44": 2.11783, - "45": 2.53232, - "46": 1.93615, - "47": 2.04965, - "48": 2.37923, - "49": 1.86396, - "50": 2.19078, - "51": 1.54546, - "52": 2.6855, - "53": 2.58466, - "54": 2.62603, - "55": 2.33101, - "56": 3.32308, - "57": 1.6526, - "58": 2.42086, - "59": 2.38312, - "60": 2.32069, - "61": 2.52682, - "62": 2.92065, - "63": 1.99964, - "64": 2.3869, - "65": 2.21157, - "66": 2.47398, - "67": 2.56848, - "68": 1.75833, - "69": 2.37531, - "70": 2.11149, - "71": 2.7953, - "72": 2.23818, - "73": 2.20853, - "74": 2.04207, - "75": 2.11843, - "76": 2.23482, - "77": 2.30573, - "78": 2.30962, - "79": 2.65893, - "80": 2.31419, - "81": 2.74547, - "82": 2.70882, - "83": 2.44411, - "84": 2.01994, - "85": 2.37118, - "86": 2.09735, - "87": 2.8617, - "88": 2.61602, - "89": 2.69622, - "90": 2.55519, - "91": 2.49755, - "92": 3.03609, - "93": 2.12677, - "94": 2.65295, - "95": 2.48188, - "96": 2.68478, - "97": 2.28276, - "98": 2.59049, - "99": 3.33845, - "100": 2.52552 + "2": 5.30901, + "3": 2.44357, + "4": 1.91681, + "5": 1.72576, + "6": 2.06436, + "7": 1.75339, + "8": 2.73322, + "9": 2.54556, + "10": 2.01375, + "11": 2.24689, + "12": 1.82849, + "13": 2.26153, + "14": 2.39938, + "15": 1.9963, + "16": 2.323, + "17": 2.93641, + "18": 2.4607, + "19": 2.47985, + "20": 2.19432, + "21": 2.59075, + "22": 1.98544, + "23": 2.61116, + "24": 2.31349, + "25": 2.7718, + "26": 2.84467, + "27": 2.2129, + "28": 2.23495, + "29": 2.14429, + "30": 2.32288, + "31": 2.06584, + "32": 3.40263, + "33": 1.92119, + "34": 1.91505, + "35": 2.71612, + "36": 2.56132, + "37": 2.19504, + "38": 1.84014, + "39": 2.88583, + "40": 2.33649, + "41": 2.41841, + "42": 3.04277, + "43": 2.22899, + "44": 2.2607, + "45": 2.85081, + "46": 2.0307, + "47": 2.2342, + "48": 2.6587, + "49": 1.91733, + "50": 2.46221, + "51": 1.35824, + "52": 2.49371, + "53": 2.55183, + "54": 2.6036, + "55": 2.16307, + "56": 3.18213, + "57": 1.67176, + "58": 2.51597, + "59": 2.18524, + "60": 2.05074, + "61": 2.58941, + "62": 2.69852, + "63": 2.00379, + "64": 2.3598, + "65": 2.22769, + "66": 2.50614, + "67": 2.55255, + "68": 1.72742, + "69": 2.39745, + "70": 2.22103, + "71": 2.78007, + "72": 2.21957, + "73": 2.28651, + "74": 2.12856, + "75": 2.16911, + "76": 2.33302, + "77": 2.30707, + "78": 2.36088, + "79": 2.79183, + "80": 2.47115, + "81": 3.18623, + "82": 2.82824, + "83": 2.62998, + "84": 2.19198, + "85": 2.27346, + "86": 2.30208, + "87": 3.09888, + "88": 2.70671, + "89": 2.7106, + "90": 2.51193, + "91": 2.47077, + "92": 2.90384, + "93": 2.0333, + "94": 2.56244, + "95": 2.48241, + "96": 2.51123, + "97": 2.15009, + "98": 2.34857, + "99": 3.02971, + "100": 2.21356 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json index 232bed8b044..112fcd07da2 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.89568, "2": 10.88438, - "3": 10.88334, - "4": 10.8784, + "3": 10.88326, + "4": 10.87852, "5": 10.88121, - "6": 10.87697, - "7": 10.88142, - "8": 10.87807, - "9": 10.8821, - "10": 10.87779, - "11": 10.87549, - "12": 10.86974, - "13": 10.86806, - "14": 10.86709, - "15": 10.83873, - "16": 10.83444, - "17": 10.84945, - "18": 10.81845, - "19": 10.84237, - "20": 10.76111, - "21": 10.75021, - "22": 10.73827, - "23": 10.72904, - "24": 10.70218, + "6": 10.87689, + "7": 10.88148, + "8": 10.87804, + "9": 10.882, + "10": 10.87788, + "11": 10.87553, + "12": 10.86972, + "13": 10.86805, + "14": 10.86702, + "15": 10.83884, + "16": 10.83449, + "17": 10.84938, + "18": 10.81843, + "19": 10.8424, + "20": 10.7611, + "21": 10.75019, + "22": 10.7383, + "23": 10.72907, + "24": 10.70225, "25": 10.69636, - "26": 10.68643, - "27": 10.65469, - "28": 10.59226, - "29": 10.55396, - "30": 10.5306, - "31": 10.52659, - "32": 10.51665, - "33": 10.4796, - "34": 10.45529, - "35": 10.45336, - "36": 10.42629, - "37": 10.39675, - "38": 10.39864, - "39": 10.36825, - "40": 10.35915, - "41": 10.33274, - "42": 10.31249, - "43": 10.29392, - "44": 10.26724, - "45": 10.27516, - "46": 10.24245, - "47": 10.22739, - "48": 10.18267, - "49": 10.18227, - "50": 10.18232, - "51": 10.18368, - "52": 10.14085, - "53": 10.14587, - "54": 10.1091, - "55": 10.08506, - "56": 10.10959, + "26": 10.68642, + "27": 10.6547, + "28": 10.59228, + "29": 10.55407, + "30": 10.53071, + "31": 10.52666, + "32": 10.51667, + "33": 10.47967, + "34": 10.45538, + "35": 10.45337, + "36": 10.42637, + "37": 10.39679, + "38": 10.39878, + "39": 10.36827, + "40": 10.35921, + "41": 10.33282, + "42": 10.31251, + "43": 10.2939, + "44": 10.26722, + "45": 10.27521, + "46": 10.24251, + "47": 10.22745, + "48": 10.1827, + "49": 10.18222, + "50": 10.18234, + "51": 10.18364, + "52": 10.14086, + "53": 10.14599, + "54": 10.10913, + "55": 10.08511, + "56": 10.10964, "57": 10.10035, - "58": 10.11463, - "59": 10.06142, - "60": 10.08134, - "61": 10.02883, - "62": 10.0024, + "58": 10.11468, + "59": 10.06147, + "60": 10.08135, + "61": 10.0289, + "62": 10.00244, "63": 10.07648, - "64": 10.03177, + "64": 10.03183, "65": 10.00985, - "66": 10.03135, - "67": 10.00962, - "68": 9.97165, - "69": 9.99531, - "70": 9.97152, - "71": 10.00044, - "72": 9.98292, - "73": 9.97408, - "74": 9.95876, - "75": 9.93551, + "66": 10.03138, + "67": 10.00963, + "68": 9.97168, + "69": 9.99535, + "70": 9.97162, + "71": 10.00045, + "72": 9.98297, + "73": 9.97412, + "74": 9.95882, + "75": 9.9356, "76": 9.96637, - "77": 9.9559, - "78": 9.91043, - "79": 9.91744, - "80": 9.93149, - "81": 9.95759, - "82": 9.88991, - "83": 9.85709, - "84": 9.79644, - "85": 9.79099, - "86": 9.88095, - "87": 9.9046, - "88": 9.87908, + "77": 9.95598, + "78": 9.91041, + "79": 9.91747, + "80": 9.93148, + "81": 9.95773, + "82": 9.88999, + "83": 9.85719, + "84": 9.79651, + "85": 9.79101, + "86": 9.88104, + "87": 9.90464, + "88": 9.87916, "89": 9.8247, - "90": 9.81244, - "91": 9.82967, - "92": 9.81771, - "93": 9.75606, - "94": 9.82372, - "95": 9.82247, - "96": 9.80489, - "97": 9.74984, - "98": 9.77682, - "99": 9.82062, - "100": 9.71291 + "90": 9.81247, + "91": 9.82969, + "92": 9.81773, + "93": 9.75609, + "94": 9.82375, + "95": 9.82251, + "96": 9.80491, + "97": 9.74995, + "98": 9.77681, + "99": 9.82066, + "100": 9.71293 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1891.0, - "2": 1889.0, - "3": 1895.0, - "4": 1909.0, - "5": 1925.0, - "6": 1911.0, - "7": 2014.0, - "8": 1827.0, - "9": 1919.0, - "10": 1881.0, + "1": 1804.0, + "2": 1707.0, + "3": 1891.0, + "4": 1824.0, + "5": 1858.0, + "6": 1869.0, + "7": 1920.0, + "8": 1661.0, + "9": 1907.0, + "10": 1826.0, "11": 1796.0, - "12": 1865.0, - "13": 1823.0, - "14": 1992.0, - "15": 1697.0, - "16": 1843.0, - "17": 1798.0, - "18": 1828.0, - "19": 1839.0, - "20": 1872.0, - "21": 1867.0, - "22": 1804.0, - "23": 1766.0, - "24": 1949.0, - "25": 1688.0, - "26": 1855.0, - "27": 1897.0, - "28": 1825.0, - "29": 1979.0, - "30": 2005.0, - "31": 2067.0, - "32": 1934.0, - "33": 2009.0, - "34": 2067.0, - "35": 2102.0, - "36": 2004.0, - "37": 2265.0, - "38": 2192.0, - "39": 2275.0, - "40": 2272.0, - "41": 2360.0, - "42": 2086.0, - "43": 2499.0, - "44": 2234.0, - "45": 2668.0, - "46": 2558.0, - "47": 2540.0, - "48": 2714.0, - "49": 2956.0, - "50": 2626.0, - "51": 2517.0, - "52": 2830.0, - "53": 2702.0, - "54": 2880.0, - "55": 2790.0, - "56": 2763.0, - "57": 2301.0, - "58": 3894.0, - "59": 3006.0, - "60": 2959.0, - "61": 2896.0, - "62": 3359.0, - "63": 3557.0, - "64": 3584.0, - "65": 2671.0, - "66": 3304.0, - "67": 4035.0, - "68": 3360.0, - "69": 3183.0, - "70": 3330.0, - "71": 3155.0, - "72": 2956.0, - "73": 3482.0, - "74": 3558.0, - "75": 3286.0, - "76": 3159.0, - "77": 3692.0, - "78": 3309.0, - "79": 3332.0, - "80": 3224.0, - "81": 3570.0, - "82": 2951.0, - "83": 3129.0, - "84": 2901.0, - "85": 2770.0, - "86": 3122.0, - "87": 3140.0, - "88": 3132.0, - "89": 3057.0, - "90": 3895.0, - "91": 2911.0, - "92": 3160.0, - "93": 3142.0, - "94": 3066.0, - "95": 3259.0, - "96": 3492.0, - "97": 3687.0, - "98": 3461.0, - "99": 3264.0, - "100": 3418.0 + "12": 1817.0, + "13": 1927.0, + "14": 1982.0, + "15": 1813.0, + "16": 1848.0, + "17": 1860.0, + "18": 1873.0, + "19": 1802.0, + "20": 1822.0, + "21": 1838.0, + "22": 1848.0, + "23": 1744.0, + "24": 1946.0, + "25": 1753.0, + "26": 1904.0, + "27": 1894.0, + "28": 1819.0, + "29": 1972.0, + "30": 1879.0, + "31": 2029.0, + "32": 1997.0, + "33": 1970.0, + "34": 2059.0, + "35": 2197.0, + "36": 2055.0, + "37": 2238.0, + "38": 2207.0, + "39": 2270.0, + "40": 2304.0, + "41": 2311.0, + "42": 2077.0, + "43": 2471.0, + "44": 2318.0, + "45": 2740.0, + "46": 2536.0, + "47": 2533.0, + "48": 2690.0, + "49": 2820.0, + "50": 2552.0, + "51": 2552.0, + "52": 2793.0, + "53": 2704.0, + "54": 2986.0, + "55": 2662.0, + "56": 2869.0, + "57": 2277.0, + "58": 3849.0, + "59": 3114.0, + "60": 2895.0, + "61": 2891.0, + "62": 3334.0, + "63": 3547.0, + "64": 3678.0, + "65": 2751.0, + "66": 3158.0, + "67": 4091.0, + "68": 3607.0, + "69": 3094.0, + "70": 3395.0, + "71": 3191.0, + "72": 3010.0, + "73": 3418.0, + "74": 3525.0, + "75": 3246.0, + "76": 3137.0, + "77": 3726.0, + "78": 3304.0, + "79": 3266.0, + "80": 3219.0, + "81": 3647.0, + "82": 3005.0, + "83": 3173.0, + "84": 2874.0, + "85": 2845.0, + "86": 3188.0, + "87": 3087.0, + "88": 3173.0, + "89": 3092.0, + "90": 3733.0, + "91": 2921.0, + "92": 3025.0, + "93": 3102.0, + "94": 3071.0, + "95": 3249.0, + "96": 3423.0, + "97": 3503.0, + "98": 3439.0, + "99": 3294.0, + "100": 3475.0 } }, "mem-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1057339904.0, - "2": 1190421504.0, - "3": 1190421504.0, - "4": 1190421504.0, - "5": 1190421504.0, - "6": 1190421504.0, - "7": 1190421504.0, - "8": 1190421504.0, - "9": 1190421504.0, - "10": 1190421504.0, - "11": 1190421504.0, - "12": 1190421504.0, - "13": 1190421504.0, - "14": 1190421504.0, - "15": 1190421504.0, - "16": 1190421504.0, - "17": 1190421504.0, - "18": 1190421504.0, - "19": 1190421504.0, - "20": 1190421504.0, - "21": 1190421504.0, - "22": 1190421504.0, - "23": 1190421504.0, - "24": 1190421504.0, - "25": 1190421504.0, - "26": 1190421504.0, - "27": 1190421504.0, - "28": 1190421504.0, - "29": 1190421504.0, - "30": 1190421504.0, - "31": 1190421504.0, - "32": 1190421504.0, - "33": 1190421504.0, - "34": 1190421504.0, - "35": 1190421504.0, - "36": 1190421504.0, - "37": 1190421504.0, - "38": 1190421504.0, - "39": 1190421504.0, - "40": 1190421504.0, - "41": 1190421504.0, - "42": 1190421504.0, - "43": 1190421504.0, - "44": 1190421504.0, - "45": 1190421504.0, - "46": 1190421504.0, - "47": 1190421504.0, - "48": 1190421504.0, - "49": 1190421504.0, - "50": 1190421504.0, - "51": 1190421504.0, - "52": 1190421504.0, - "53": 1190421504.0, - "54": 1190421504.0, - "55": 1190421504.0, - "56": 1190421504.0, - "57": 1190421504.0, - "58": 1190421504.0, - "59": 1190421504.0, - "60": 1190421504.0, - "61": 1190421504.0, - "62": 1190421504.0, - "63": 1190421504.0, - "64": 1190421504.0, - "65": 1190421504.0, - "66": 1190421504.0, - "67": 1190421504.0, - "68": 1190421504.0, - "69": 1190421504.0, - "70": 1190421504.0, - "71": 1190421504.0, - "72": 1190421504.0, - "73": 1190421504.0, - "74": 1190421504.0, - "75": 1190421504.0, - "76": 1190421504.0, - "77": 1190421504.0, - "78": 1190421504.0, - "79": 1190421504.0, - "80": 1190421504.0, - "81": 1190421504.0, - "82": 1190421504.0, - "83": 1190421504.0, - "84": 1190421504.0, - "85": 1190421504.0, - "86": 1190421504.0, - "87": 1190421504.0, - "88": 1190421504.0, - "89": 1190421504.0, - "90": 1190421504.0, - "91": 1190421504.0, - "92": 1190421504.0, - "93": 1190421504.0, - "94": 1190421504.0, - "95": 1190421504.0, - "96": 1190421504.0, - "97": 1190421504.0, - "98": 1190421504.0, - "99": 1190421504.0, - "100": 1190421504.0 + "1": 1053145600.0, + "2": 1185178624.0, + "3": 1185178624.0, + "4": 1185178624.0, + "5": 1185178624.0, + "6": 1185178624.0, + "7": 1185178624.0, + "8": 1185178624.0, + "9": 1185178624.0, + "10": 1185178624.0, + "11": 1185178624.0, + "12": 1185178624.0, + "13": 1185178624.0, + "14": 1185178624.0, + "15": 1185178624.0, + "16": 1185178624.0, + "17": 1185178624.0, + "18": 1185178624.0, + "19": 1185178624.0, + "20": 1185178624.0, + "21": 1185178624.0, + "22": 1185178624.0, + "23": 1185178624.0, + "24": 1185178624.0, + "25": 1185178624.0, + "26": 1185178624.0, + "27": 1185178624.0, + "28": 1185178624.0, + "29": 1185178624.0, + "30": 1185178624.0, + "31": 1185178624.0, + "32": 1185178624.0, + "33": 1185178624.0, + "34": 1185178624.0, + "35": 1185178624.0, + "36": 1185178624.0, + "37": 1185178624.0, + "38": 1185178624.0, + "39": 1185178624.0, + "40": 1185178624.0, + "41": 1185178624.0, + "42": 1185178624.0, + "43": 1185178624.0, + "44": 1185178624.0, + "45": 1185178624.0, + "46": 1185178624.0, + "47": 1185178624.0, + "48": 1185178624.0, + "49": 1185178624.0, + "50": 1185178624.0, + "51": 1185178624.0, + "52": 1185178624.0, + "53": 1185178624.0, + "54": 1185178624.0, + "55": 1185178624.0, + "56": 1185178624.0, + "57": 1185178624.0, + "58": 1185178624.0, + "59": 1185178624.0, + "60": 1185178624.0, + "61": 1185178624.0, + "62": 1185178624.0, + "63": 1185178624.0, + "64": 1185178624.0, + "65": 1185178624.0, + "66": 1185178624.0, + "67": 1185178624.0, + "68": 1185178624.0, + "69": 1185178624.0, + "70": 1185178624.0, + "71": 1185178624.0, + "72": 1185178624.0, + "73": 1185178624.0, + "74": 1185178624.0, + "75": 1185178624.0, + "76": 1185178624.0, + "77": 1185178624.0, + "78": 1185178624.0, + "79": 1185178624.0, + "80": 1185178624.0, + "81": 1185178624.0, + "82": 1185178624.0, + "83": 1185178624.0, + "84": 1185178624.0, + "85": 1185178624.0, + "86": 1185178624.0, + "87": 1185178624.0, + "88": 1185178624.0, + "89": 1185178624.0, + "90": 1185178624.0, + "91": 1185178624.0, + "92": 1185178624.0, + "93": 1185178624.0, + "94": 1185178624.0, + "95": 1185178624.0, + "96": 1185178624.0, + "97": 1185178624.0, + "98": 1185178624.0, + "99": 1185178624.0, + "100": 1185178624.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.93212, - "3": 0.4867, - "4": 0.4747, - "5": 0.47507, - "6": 0.47592, - "7": 0.48344, - "8": 0.47802, - "9": 0.47959, - "10": 0.47727, - "11": 0.4783, - "12": 0.47575, - "13": 0.4795, - "14": 0.47762, - "15": 0.4805, - "16": 0.47661, - "17": 0.4758, - "18": 0.47928, - "19": 0.48965, - "20": 0.47908, - "21": 0.47724, - "22": 0.48039, - "23": 0.48086, - "24": 0.47743, - "25": 0.48716, - "26": 0.47952, - "27": 0.47855, - "28": 0.48454, - "29": 0.484, - "30": 0.47913, - "31": 0.47855, - "32": 0.48162, - "33": 0.48076, - "34": 0.4795, - "35": 0.4815, - "36": 0.47988, - "37": 0.481, - "38": 0.47885, - "39": 0.48175, - "40": 0.47684, - "41": 0.48106, - "42": 0.47776, - "43": 0.47695, - "44": 0.47797, - "45": 0.48075, - "46": 0.48073, - "47": 0.47964, - "48": 0.48021, - "49": 0.47813, - "50": 0.48082, - "51": 0.66597, - "52": 0.52351, - "53": 0.47646, - "54": 0.47743, - "55": 0.47846, - "56": 0.47847, - "57": 0.47533, - "58": 0.47594, - "59": 0.47691, - "60": 0.47669, - "61": 0.47512, - "62": 0.47864, - "63": 0.48003, - "64": 0.48478, - "65": 0.47948, - "66": 0.47936, - "67": 0.48064, - "68": 0.48013, - "69": 0.47966, - "70": 0.48094, - "71": 0.47735, - "72": 0.47731, - "73": 0.47692, - "74": 0.47916, - "75": 0.47978, - "76": 0.48233, - "77": 0.4809, - "78": 0.48115, - "79": 0.47935, - "80": 0.47807, - "81": 0.47746, - "82": 0.48121, - "83": 0.48199, - "84": 0.48397, - "85": 0.48206, - "86": 0.48051, - "87": 0.48093, - "88": 0.4814, - "89": 0.48224, - "90": 0.48608, - "91": 0.48635, - "92": 0.48448, - "93": 0.48512, - "94": 0.48328, - "95": 0.48162, - "96": 0.48286, - "97": 0.48224, - "98": 0.48128, - "99": 0.47755, - "100": 0.47843 + "2": 6.01093, + "3": 0.44954, + "4": 0.4012, + "5": 0.40603, + "6": 0.40498, + "7": 0.40968, + "8": 0.40977, + "9": 0.40063, + "10": 0.40296, + "11": 0.40721, + "12": 0.40472, + "13": 0.40674, + "14": 0.40079, + "15": 0.40361, + "16": 0.40826, + "17": 0.40286, + "18": 0.40169, + "19": 0.4041, + "20": 0.40483, + "21": 0.40457, + "22": 0.4087, + "23": 0.40329, + "24": 0.40421, + "25": 0.4002, + "26": 0.40406, + "27": 0.40312, + "28": 0.40188, + "29": 0.39936, + "30": 0.39985, + "31": 0.40327, + "32": 0.4067, + "33": 0.41283, + "34": 0.40735, + "35": 0.40232, + "36": 0.4045, + "37": 0.41308, + "38": 0.40553, + "39": 0.39906, + "40": 0.40266, + "41": 0.40088, + "42": 0.39992, + "43": 0.39717, + "44": 0.39905, + "45": 0.39847, + "46": 0.40111, + "47": 0.3989, + "48": 0.40238, + "49": 0.40391, + "50": 0.40641, + "51": 0.63301, + "52": 0.46934, + "53": 0.40029, + "54": 0.40061, + "55": 0.40349, + "56": 0.40544, + "57": 0.40292, + "58": 0.40611, + "59": 0.40578, + "60": 0.40208, + "61": 0.40634, + "62": 0.40232, + "63": 0.40539, + "64": 0.40354, + "65": 0.40053, + "66": 0.40506, + "67": 0.40307, + "68": 0.40468, + "69": 0.40349, + "70": 0.4073, + "71": 0.4059, + "72": 0.40536, + "73": 0.40435, + "74": 0.40922, + "75": 0.40764, + "76": 0.7658, + "77": 0.399, + "78": 0.40056, + "79": 0.39794, + "80": 0.39514, + "81": 0.3978, + "82": 0.39955, + "83": 0.40033, + "84": 0.40133, + "85": 0.40355, + "86": 0.40301, + "87": 0.40094, + "88": 0.40071, + "89": 0.40235, + "90": 0.40106, + "91": 0.40144, + "92": 0.40469, + "93": 0.40026, + "94": 0.40348, + "95": 0.40253, + "96": 0.40495, + "97": 0.40546, + "98": 0.40589, + "99": 0.41488, + "100": 0.40564 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json index e317e183d08..918b01c6381 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89706, - "2": 10.88994, - "3": 10.8831, - "4": 10.88988, - "5": 10.88253, - "6": 10.8874, - "7": 10.88758, - "8": 10.88283, - "9": 10.88381, - "10": 10.88124, - "11": 10.87942, - "12": 10.87716, - "13": 10.87392, - "14": 10.86886, - "15": 10.84732, - "16": 10.83848, - "17": 10.83887, - "18": 10.82811, - "19": 10.83477, - "20": 10.76248, - "21": 10.74325, - "22": 10.72764, - "23": 10.72152, - "24": 10.70346, - "25": 10.69674, - "26": 10.68042, - "27": 10.654, - "28": 10.57729, - "29": 10.55345, - "30": 10.52537, - "31": 10.52801, - "32": 10.51618, - "33": 10.47008, - "34": 10.4406, - "35": 10.44039, - "36": 10.42417, - "37": 10.39196, - "38": 10.39172, - "39": 10.35867, - "40": 10.34947, - "41": 10.32488, - "42": 10.30815, - "43": 10.28559, + "1": 10.89695, + "2": 10.88983, + "3": 10.88313, + "4": 10.88984, + "5": 10.88257, + "6": 10.88737, + "7": 10.88753, + "8": 10.88276, + "9": 10.88373, + "10": 10.8813, + "11": 10.87945, + "12": 10.87721, + "13": 10.87391, + "14": 10.86889, + "15": 10.84742, + "16": 10.8385, + "17": 10.83891, + "18": 10.82804, + "19": 10.83469, + "20": 10.76256, + "21": 10.74328, + "22": 10.72762, + "23": 10.72153, + "24": 10.70345, + "25": 10.69669, + "26": 10.68047, + "27": 10.65412, + "28": 10.57731, + "29": 10.55353, + "30": 10.52543, + "31": 10.52806, + "32": 10.51614, + "33": 10.47016, + "34": 10.44068, + "35": 10.44041, + "36": 10.42432, + "37": 10.392, + "38": 10.39181, + "39": 10.35863, + "40": 10.34956, + "41": 10.32492, + "42": 10.30831, + "43": 10.28562, "44": 10.26537, - "45": 10.26838, - "46": 10.23866, - "47": 10.22366, + "45": 10.2684, + "46": 10.2387, + "47": 10.22376, "48": 10.17493, - "49": 10.18427, - "50": 10.17934, - "51": 10.18127, + "49": 10.18431, + "50": 10.17937, + "51": 10.18132, "52": 10.13579, - "53": 10.13436, - "54": 10.1078, - "55": 10.0851, - "56": 10.10727, - "57": 10.10421, - "58": 10.11001, - "59": 10.04654, - "60": 10.07278, - "61": 10.02695, - "62": 10.00112, - "63": 10.06931, - "64": 10.02132, - "65": 9.98311, - "66": 10.02645, - "67": 10.0049, - "68": 9.96999, - "69": 9.98961, - "70": 9.96903, - "71": 9.99459, - "72": 9.98044, - "73": 9.96536, - "74": 9.95995, - "75": 9.9297, - "76": 9.95438, - "77": 9.95044, + "53": 10.13446, + "54": 10.1079, + "55": 10.08516, + "56": 10.10725, + "57": 10.10425, + "58": 10.11006, + "59": 10.04658, + "60": 10.07284, + "61": 10.02703, + "62": 10.00117, + "63": 10.06929, + "64": 10.02141, + "65": 9.98312, + "66": 10.02646, + "67": 10.00489, + "68": 9.97007, + "69": 9.98965, + "70": 9.96905, + "71": 9.99468, + "72": 9.98046, + "73": 9.96537, + "74": 9.95994, + "75": 9.92969, + "76": 9.95444, + "77": 9.95046, "78": 9.90492, "79": 9.90719, - "80": 9.92276, - "81": 9.94802, - "82": 9.88462, - "83": 9.84481, - "84": 9.78362, - "85": 9.77466, - "86": 9.87595, - "87": 9.90368, - "88": 9.87789, - "89": 9.82816, - "90": 9.81308, - "91": 9.82262, - "92": 9.81558, - "93": 9.75135, - "94": 9.82019, - "95": 9.80988, - "96": 9.79633, - "97": 9.7464, - "98": 9.76836, - "99": 9.82002, - "100": 9.70572 + "80": 9.92282, + "81": 9.94809, + "82": 9.88468, + "83": 9.84487, + "84": 9.78366, + "85": 9.77469, + "86": 9.87604, + "87": 9.90369, + "88": 9.87787, + "89": 9.82814, + "90": 9.81311, + "91": 9.82266, + "92": 9.81563, + "93": 9.7514, + "94": 9.82022, + "95": 9.80985, + "96": 9.79637, + "97": 9.74636, + "98": 9.76834, + "99": 9.82001, + "100": 9.70575 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1873.0, - "2": 1703.0, - "3": 1801.0, - "4": 1750.0, - "5": 1905.0, - "6": 1785.0, - "7": 2037.0, - "8": 1770.0, - "9": 1855.0, - "10": 1768.0, - "11": 1853.0, - "12": 1789.0, - "13": 1844.0, - "14": 1998.0, - "15": 1666.0, - "16": 1838.0, - "17": 1864.0, - "18": 1877.0, - "19": 1721.0, - "20": 1768.0, - "21": 1836.0, - "22": 1810.0, - "23": 1805.0, - "24": 1852.0, - "25": 1700.0, - "26": 1866.0, - "27": 1857.0, - "28": 1983.0, - "29": 1960.0, - "30": 1873.0, - "31": 1994.0, - "32": 1968.0, - "33": 2062.0, - "34": 2072.0, - "35": 2136.0, - "36": 2050.0, - "37": 2255.0, - "38": 2187.0, - "39": 2281.0, - "40": 2282.0, - "41": 2432.0, - "42": 2103.0, - "43": 2372.0, - "44": 2275.0, - "45": 2547.0, - "46": 2458.0, - "47": 2531.0, - "48": 2675.0, - "49": 2879.0, - "50": 2643.0, - "51": 2682.0, - "52": 2750.0, - "53": 2650.0, - "54": 2927.0, - "55": 2782.0, - "56": 2814.0, - "57": 2300.0, - "58": 3764.0, - "59": 3015.0, - "60": 2825.0, - "61": 2787.0, - "62": 3072.0, - "63": 3348.0, - "64": 3672.0, - "65": 2758.0, - "66": 3305.0, - "67": 3796.0, - "68": 3510.0, - "69": 3149.0, - "70": 3372.0, - "71": 3109.0, - "72": 3044.0, - "73": 3476.0, - "74": 3314.0, - "75": 3212.0, - "76": 3352.0, - "77": 3845.0, - "78": 3174.0, - "79": 3428.0, - "80": 3290.0, - "81": 3654.0, - "82": 3055.0, - "83": 3112.0, - "84": 2886.0, - "85": 2844.0, - "86": 3160.0, - "87": 2896.0, - "88": 3019.0, - "89": 3058.0, - "90": 3962.0, - "91": 3083.0, - "92": 2849.0, - "93": 3194.0, - "94": 3174.0, - "95": 3180.0, - "96": 3447.0, - "97": 3513.0, - "98": 3294.0, - "99": 3156.0, - "100": 3173.0 + "1": 1891.0, + "2": 1772.0, + "3": 1833.0, + "4": 1776.0, + "5": 1784.0, + "6": 1757.0, + "7": 2022.0, + "8": 1721.0, + "9": 1845.0, + "10": 1772.0, + "11": 1753.0, + "12": 1818.0, + "13": 1835.0, + "14": 2026.0, + "15": 1720.0, + "16": 1817.0, + "17": 1785.0, + "18": 1833.0, + "19": 1752.0, + "20": 1821.0, + "21": 1927.0, + "22": 1860.0, + "23": 1831.0, + "24": 1919.0, + "25": 1836.0, + "26": 1790.0, + "27": 1928.0, + "28": 1950.0, + "29": 1984.0, + "30": 1786.0, + "31": 2116.0, + "32": 2023.0, + "33": 2116.0, + "34": 2014.0, + "35": 2193.0, + "36": 2059.0, + "37": 2253.0, + "38": 2209.0, + "39": 2236.0, + "40": 2332.0, + "41": 2398.0, + "42": 2056.0, + "43": 2522.0, + "44": 2278.0, + "45": 2596.0, + "46": 2487.0, + "47": 2677.0, + "48": 2718.0, + "49": 2900.0, + "50": 2634.0, + "51": 2567.0, + "52": 2804.0, + "53": 2792.0, + "54": 2945.0, + "55": 2757.0, + "56": 2764.0, + "57": 2211.0, + "58": 3602.0, + "59": 3054.0, + "60": 2947.0, + "61": 2765.0, + "62": 3114.0, + "63": 3345.0, + "64": 3654.0, + "65": 2747.0, + "66": 3186.0, + "67": 3872.0, + "68": 3559.0, + "69": 3093.0, + "70": 3290.0, + "71": 3191.0, + "72": 3101.0, + "73": 3496.0, + "74": 3306.0, + "75": 3337.0, + "76": 3347.0, + "77": 3844.0, + "78": 3209.0, + "79": 3308.0, + "80": 3272.0, + "81": 3617.0, + "82": 2967.0, + "83": 3186.0, + "84": 2884.0, + "85": 2773.0, + "86": 3289.0, + "87": 2849.0, + "88": 3017.0, + "89": 3186.0, + "90": 3781.0, + "91": 3089.0, + "92": 2931.0, + "93": 3300.0, + "94": 3308.0, + "95": 3303.0, + "96": 3432.0, + "97": 3467.0, + "98": 3331.0, + "99": 3041.0, + "100": 3055.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.96215, - "3": 0.40447, - "4": 0.40615, - "5": 0.4052, - "6": 0.3984, - "7": 0.40189, - "8": 0.40565, - "9": 0.4015, - "10": 0.39982, - "11": 0.40336, - "12": 0.40808, - "13": 0.40287, - "14": 0.40086, - "15": 0.40962, - "16": 0.40693, - "17": 0.39911, - "18": 0.40497, - "19": 0.39981, - "20": 0.39833, - "21": 0.40002, - "22": 0.40553, - "23": 0.39778, - "24": 0.3997, - "25": 0.40293, - "26": 0.39987, - "27": 0.39787, - "28": 0.39995, - "29": 0.40733, - "30": 0.40184, - "31": 0.40281, - "32": 0.4007, - "33": 0.40099, - "34": 0.39968, - "35": 0.40118, - "36": 0.40042, - "37": 0.3973, - "38": 0.39656, - "39": 0.39681, - "40": 0.3995, - "41": 0.39914, - "42": 0.39822, - "43": 0.39987, - "44": 0.39748, - "45": 0.39883, - "46": 0.4004, - "47": 0.41492, - "48": 0.4007, - "49": 0.40707, - "50": 0.39954, - "51": 0.53848, - "52": 0.43866, - "53": 0.39439, - "54": 0.39643, - "55": 0.40161, - "56": 0.39595, - "57": 0.40415, - "58": 0.39487, - "59": 0.39625, - "60": 0.4016, - "61": 0.39878, - "62": 0.40098, - "63": 0.39832, - "64": 0.40379, - "65": 0.39826, - "66": 0.40045, - "67": 0.39726, - "68": 0.39694, - "69": 0.39541, - "70": 0.39651, - "71": 0.40066, - "72": 0.39547, - "73": 0.40045, - "74": 0.39279, - "75": 0.39432, - "76": 0.40002, - "77": 0.39686, - "78": 0.40338, - "79": 0.39855, - "80": 0.39739, - "81": 0.39514, - "82": 0.39807, - "83": 0.39541, - "84": 0.40009, - "85": 0.39672, - "86": 0.409, - "87": 0.3964, - "88": 0.40293, - "89": 0.40117, - "90": 0.40761, - "91": 0.40043, - "92": 0.40507, - "93": 0.40323, - "94": 0.40287, - "95": 0.40399, - "96": 0.39454, - "97": 0.3944, - "98": 0.39944, - "99": 0.39466, - "100": 0.39438 + "2": 4.11988, + "3": 0.43278, + "4": 0.42362, + "5": 0.42776, + "6": 0.43785, + "7": 0.42516, + "8": 0.42654, + "9": 0.42467, + "10": 0.43473, + "11": 0.42563, + "12": 0.42209, + "13": 0.42686, + "14": 0.43097, + "15": 0.43208, + "16": 0.42874, + "17": 0.42807, + "18": 0.42549, + "19": 0.42402, + "20": 0.42157, + "21": 0.42171, + "22": 0.42504, + "23": 0.41896, + "24": 0.42124, + "25": 0.42736, + "26": 0.42117, + "27": 0.42565, + "28": 0.42879, + "29": 0.41271, + "30": 0.42097, + "31": 0.41608, + "32": 0.41302, + "33": 0.41372, + "34": 0.42656, + "35": 0.42275, + "36": 0.42091, + "37": 0.42045, + "38": 0.4235, + "39": 0.41574, + "40": 0.4157, + "41": 0.42612, + "42": 0.41967, + "43": 0.41771, + "44": 0.41148, + "45": 0.41636, + "46": 0.42571, + "47": 0.41243, + "48": 0.41797, + "49": 0.41365, + "50": 0.41295, + "51": 0.50994, + "52": 1.13947, + "53": 0.42314, + "54": 0.41959, + "55": 0.42487, + "56": 0.41381, + "57": 0.4207, + "58": 0.41425, + "59": 0.41075, + "60": 0.4214, + "61": 0.41151, + "62": 0.42302, + "63": 0.41797, + "64": 0.41488, + "65": 0.41713, + "66": 0.41725, + "67": 0.42138, + "68": 0.41569, + "69": 0.41919, + "70": 0.41415, + "71": 0.41272, + "72": 0.41801, + "73": 0.4113, + "74": 0.4161, + "75": 0.41083, + "76": 0.41351, + "77": 0.42248, + "78": 0.41589, + "79": 0.41989, + "80": 0.41401, + "81": 0.41636, + "82": 0.4123, + "83": 0.41352, + "84": 0.41334, + "85": 0.41215, + "86": 0.41122, + "87": 0.41344, + "88": 0.41657, + "89": 0.41347, + "90": 0.41331, + "91": 0.41393, + "92": 0.4107, + "93": 0.4106, + "94": 0.41142, + "95": 0.41783, + "96": 0.41123, + "97": 0.41381, + "98": 0.41357, + "99": 0.41011, + "100": 0.41482 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json index 63039c03925..4c74407856c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89534, - "2": 10.88313, - "3": 10.88058, - "4": 10.8822, - "5": 10.88321, - "6": 10.87484, - "7": 10.87757, - "8": 10.88301, - "9": 10.88054, - "10": 10.87887, - "11": 10.87541, - "12": 10.86979, - "13": 10.86712, - "14": 10.86521, - "15": 10.84124, - "16": 10.83552, - "17": 10.84586, - "18": 10.82014, - "19": 10.84135, + "1": 10.89522, + "2": 10.88315, + "3": 10.88059, + "4": 10.88226, + "5": 10.88328, + "6": 10.87479, + "7": 10.87755, + "8": 10.883, + "9": 10.88055, + "10": 10.87891, + "11": 10.87538, + "12": 10.86983, + "13": 10.86708, + "14": 10.86523, + "15": 10.8413, + "16": 10.83556, + "17": 10.84593, + "18": 10.82029, + "19": 10.84133, "20": 10.76033, "21": 10.7515, - "22": 10.73896, - "23": 10.72982, - "24": 10.70361, - "25": 10.69585, - "26": 10.68595, - "27": 10.65365, - "28": 10.592, - "29": 10.55593, - "30": 10.53219, - "31": 10.52302, - "32": 10.51797, - "33": 10.47935, - "34": 10.45411, - "35": 10.454, - "36": 10.42698, - "37": 10.39699, - "38": 10.40023, - "39": 10.36898, - "40": 10.35931, - "41": 10.33342, - "42": 10.31186, - "43": 10.29213, - "44": 10.26934, - "45": 10.27395, - "46": 10.24145, - "47": 10.227, - "48": 10.18305, - "49": 10.18406, - "50": 10.18231, - "51": 10.18171, - "52": 10.14054, - "53": 10.14639, - "54": 10.11019, - "55": 10.08362, - "56": 10.11023, - "57": 10.09879, - "58": 10.11422, - "59": 10.06263, - "60": 10.08229, - "61": 10.03027, - "62": 10.00397, - "63": 10.07492, - "64": 10.03142, - "65": 10.00976, - "66": 10.03138, - "67": 10.00902, - "68": 9.97028, - "69": 9.99473, - "70": 9.97175, - "71": 10.00142, - "72": 9.98221, - "73": 9.97407, - "74": 9.95811, - "75": 9.93459, - "76": 9.96651, - "77": 9.9566, - "78": 9.91141, - "79": 9.91914, - "80": 9.9304, - "81": 9.95666, - "82": 9.88909, - "83": 9.85746, - "84": 9.79774, - "85": 9.78932, - "86": 9.87974, - "87": 9.90388, - "88": 9.87781, - "89": 9.82415, - "90": 9.81292, - "91": 9.8278, - "92": 9.81697, - "93": 9.75605, - "94": 9.82456, - "95": 9.8217, - "96": 9.80484, - "97": 9.75029, - "98": 9.77497, - "99": 9.82037, - "100": 9.7128 + "22": 10.73902, + "23": 10.7299, + "24": 10.70356, + "25": 10.6959, + "26": 10.68597, + "27": 10.65363, + "28": 10.59201, + "29": 10.55599, + "30": 10.53236, + "31": 10.52309, + "32": 10.51801, + "33": 10.47929, + "34": 10.45416, + "35": 10.45393, + "36": 10.42704, + "37": 10.39711, + "38": 10.40027, + "39": 10.36904, + "40": 10.35941, + "41": 10.33345, + "42": 10.31197, + "43": 10.29218, + "44": 10.26941, + "45": 10.27403, + "46": 10.24156, + "47": 10.22712, + "48": 10.18309, + "49": 10.18407, + "50": 10.18236, + "51": 10.18179, + "52": 10.14057, + "53": 10.14642, + "54": 10.11018, + "55": 10.08366, + "56": 10.11032, + "57": 10.09881, + "58": 10.11426, + "59": 10.06266, + "60": 10.08236, + "61": 10.03029, + "62": 10.00394, + "63": 10.07497, + "64": 10.03145, + "65": 10.00975, + "66": 10.03141, + "67": 10.00898, + "68": 9.97035, + "69": 9.99478, + "70": 9.9718, + "71": 10.00151, + "72": 9.98226, + "73": 9.97409, + "74": 9.95817, + "75": 9.93464, + "76": 9.96655, + "77": 9.95664, + "78": 9.91146, + "79": 9.91925, + "80": 9.93044, + "81": 9.95669, + "82": 9.88913, + "83": 9.8575, + "84": 9.79778, + "85": 9.78935, + "86": 9.8798, + "87": 9.90392, + "88": 9.87786, + "89": 9.82422, + "90": 9.81295, + "91": 9.82784, + "92": 9.81705, + "93": 9.75609, + "94": 9.82461, + "95": 9.82176, + "96": 9.80487, + "97": 9.75041, + "98": 9.77503, + "99": 9.82042, + "100": 9.71285 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1879.0, - "2": 1827.0, - "3": 1946.0, - "4": 1786.0, - "5": 1934.0, - "6": 1687.0, - "7": 1969.0, - "8": 1907.0, - "9": 1922.0, - "10": 1838.0, - "11": 1809.0, - "12": 1821.0, - "13": 1859.0, - "14": 2036.0, - "15": 1774.0, - "16": 1853.0, - "17": 1932.0, - "18": 1854.0, - "19": 1797.0, - "20": 1814.0, - "21": 1858.0, - "22": 1779.0, - "23": 1827.0, - "24": 1837.0, - "25": 1757.0, - "26": 1871.0, - "27": 1867.0, - "28": 1949.0, - "29": 2044.0, - "30": 1926.0, - "31": 2023.0, - "32": 1969.0, - "33": 2007.0, - "34": 2111.0, - "35": 2177.0, - "36": 2119.0, - "37": 2240.0, - "38": 2178.0, - "39": 2220.0, - "40": 2368.0, - "41": 2364.0, - "42": 2047.0, - "43": 2445.0, - "44": 2287.0, - "45": 2599.0, - "46": 2556.0, - "47": 2488.0, - "48": 2719.0, - "49": 2822.0, - "50": 2595.0, - "51": 2602.0, - "52": 2804.0, - "53": 2786.0, - "54": 3027.0, - "55": 2718.0, - "56": 2783.0, - "57": 2218.0, - "58": 3910.0, - "59": 3107.0, - "60": 3134.0, - "61": 2834.0, - "62": 3274.0, - "63": 3534.0, - "64": 3542.0, - "65": 2765.0, - "66": 3175.0, - "67": 4116.0, - "68": 3516.0, - "69": 3088.0, - "70": 3237.0, - "71": 3162.0, - "72": 3061.0, - "73": 3470.0, - "74": 3406.0, - "75": 3253.0, - "76": 3291.0, - "77": 3771.0, - "78": 3318.0, - "79": 3240.0, - "80": 3152.0, - "81": 3521.0, - "82": 2878.0, - "83": 3065.0, - "84": 2924.0, - "85": 2831.0, - "86": 3152.0, - "87": 3004.0, - "88": 3149.0, - "89": 3162.0, - "90": 3938.0, - "91": 2951.0, - "92": 2982.0, - "93": 3147.0, - "94": 3175.0, - "95": 3284.0, - "96": 3419.0, - "97": 3522.0, - "98": 3569.0, - "99": 3208.0, - "100": 3476.0 + "1": 1854.0, + "2": 1858.0, + "3": 1941.0, + "4": 1732.0, + "5": 1920.0, + "6": 1844.0, + "7": 2008.0, + "8": 1823.0, + "9": 1885.0, + "10": 1800.0, + "11": 1804.0, + "12": 1833.0, + "13": 1890.0, + "14": 2028.0, + "15": 1803.0, + "16": 1866.0, + "17": 1906.0, + "18": 1839.0, + "19": 1814.0, + "20": 1782.0, + "21": 1916.0, + "22": 1728.0, + "23": 1825.0, + "24": 1933.0, + "25": 1798.0, + "26": 1881.0, + "27": 1897.0, + "28": 1965.0, + "29": 2001.0, + "30": 1824.0, + "31": 2049.0, + "32": 2032.0, + "33": 2033.0, + "34": 2158.0, + "35": 2254.0, + "36": 2053.0, + "37": 2258.0, + "38": 2168.0, + "39": 2280.0, + "40": 2294.0, + "41": 2441.0, + "42": 2024.0, + "43": 2524.0, + "44": 2292.0, + "45": 2714.0, + "46": 2572.0, + "47": 2539.0, + "48": 2637.0, + "49": 2847.0, + "50": 2702.0, + "51": 2607.0, + "52": 2808.0, + "53": 2784.0, + "54": 2930.0, + "55": 2769.0, + "56": 2776.0, + "57": 2284.0, + "58": 3904.0, + "59": 3072.0, + "60": 3099.0, + "61": 2752.0, + "62": 3299.0, + "63": 3450.0, + "64": 3634.0, + "65": 2732.0, + "66": 3266.0, + "67": 3987.0, + "68": 3531.0, + "69": 3176.0, + "70": 3288.0, + "71": 3101.0, + "72": 3013.0, + "73": 3410.0, + "74": 3499.0, + "75": 3242.0, + "76": 3230.0, + "77": 3823.0, + "78": 3356.0, + "79": 3356.0, + "80": 3297.0, + "81": 3544.0, + "82": 2859.0, + "83": 3021.0, + "84": 2948.0, + "85": 2836.0, + "86": 3273.0, + "87": 3039.0, + "88": 3176.0, + "89": 3121.0, + "90": 3813.0, + "91": 2815.0, + "92": 3053.0, + "93": 3093.0, + "94": 3149.0, + "95": 3211.0, + "96": 3548.0, + "97": 3562.0, + "98": 3617.0, + "99": 3132.0, + "100": 3502.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.08928, - "3": 2.4142, - "4": 2.40177, - "5": 1.88805, - "6": 2.50569, - "7": 1.94968, - "8": 2.7794, - "9": 2.94828, - "10": 2.50798, - "11": 3.11261, - "12": 2.80493, - "13": 2.73852, - "14": 3.24157, - "15": 2.46907, - "16": 3.11593, - "17": 3.19334, - "18": 2.31346, - "19": 2.65344, - "20": 2.29061, - "21": 2.72492, - "22": 1.98624, - "23": 2.47979, - "24": 2.1021, - "25": 2.95103, - "26": 3.02023, - "27": 2.29962, - "28": 2.49912, - "29": 2.11753, - "30": 2.24013, - "31": 2.15314, - "32": 4.25568, - "33": 2.69117, - "34": 3.0567, - "35": 3.68884, - "36": 3.19579, - "37": 2.48692, - "38": 2.53056, - "39": 3.09628, - "40": 2.4614, - "41": 2.51819, - "42": 3.1751, - "43": 2.28582, - "44": 2.47582, - "45": 2.82828, - "46": 2.2188, - "47": 2.38142, - "48": 2.77833, - "49": 1.88475, - "50": 2.2008, - "51": 1.57454, - "52": 2.43769, - "53": 2.63045, - "54": 2.19334, - "55": 2.50425, - "56": 3.24638, - "57": 1.69979, - "58": 2.66862, - "59": 2.11892, - "60": 2.45527, - "61": 2.526, - "62": 2.92742, - "63": 2.05047, - "64": 2.40871, - "65": 2.37633, - "66": 2.68518, - "67": 2.59126, - "68": 1.8339, - "69": 2.6468, - "70": 2.2852, - "71": 2.88407, - "72": 2.23295, - "73": 2.2724, - "74": 1.9465, - "75": 1.89306, - "76": 2.1995, - "77": 2.26172, - "78": 2.20307, - "79": 2.62622, - "80": 2.56017, - "81": 2.86228, - "82": 2.92284, - "83": 2.56572, - "84": 2.74261, - "85": 2.88594, - "86": 2.20958, - "87": 3.25945, - "88": 2.60166, - "89": 2.6639, - "90": 3.45798, - "91": 3.13275, - "92": 3.26408, - "93": 2.15378, - "94": 2.76238, - "95": 2.43012, - "96": 2.50507, - "97": 2.23095, - "98": 2.50054, - "99": 3.59538, - "100": 2.39324 + "2": 5.15765, + "3": 2.64426, + "4": 2.49545, + "5": 2.00961, + "6": 2.58214, + "7": 2.04909, + "8": 2.69914, + "9": 2.86306, + "10": 2.32072, + "11": 2.52837, + "12": 1.9811, + "13": 2.12326, + "14": 2.37859, + "15": 1.90018, + "16": 2.14032, + "17": 2.99677, + "18": 2.20375, + "19": 2.40322, + "20": 2.18886, + "21": 2.59596, + "22": 1.99184, + "23": 2.58606, + "24": 2.02942, + "25": 2.78528, + "26": 2.96628, + "27": 2.22195, + "28": 2.29649, + "29": 2.01222, + "30": 2.19611, + "31": 2.01995, + "32": 3.3874, + "33": 1.97273, + "34": 1.92423, + "35": 2.80117, + "36": 2.69282, + "37": 2.32018, + "38": 1.85783, + "39": 2.77066, + "40": 2.29111, + "41": 2.34997, + "42": 3.02386, + "43": 2.14893, + "44": 2.36259, + "45": 2.87394, + "46": 2.11643, + "47": 2.27063, + "48": 2.58991, + "49": 2.00147, + "50": 2.68782, + "51": 1.85051, + "52": 2.47405, + "53": 2.44366, + "54": 2.52167, + "55": 2.28638, + "56": 3.03065, + "57": 1.58805, + "58": 2.40785, + "59": 2.06052, + "60": 2.13093, + "61": 2.6438, + "62": 2.76045, + "63": 1.9227, + "64": 2.40176, + "65": 2.18424, + "66": 2.42663, + "67": 2.43148, + "68": 1.69605, + "69": 2.34154, + "70": 2.11248, + "71": 2.50911, + "72": 1.92958, + "73": 1.97172, + "74": 1.89918, + "75": 1.92618, + "76": 2.04857, + "77": 2.00447, + "78": 2.09197, + "79": 2.54993, + "80": 2.26054, + "81": 2.52855, + "82": 2.45496, + "83": 2.37072, + "84": 1.9891, + "85": 2.09655, + "86": 2.09492, + "87": 2.75111, + "88": 2.40901, + "89": 2.1762, + "90": 2.06189, + "91": 2.11953, + "92": 2.51481, + "93": 1.74384, + "94": 2.32888, + "95": 1.93064, + "96": 2.07576, + "97": 1.90374, + "98": 2.12555, + "99": 2.81842, + "100": 2.03571 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json index ad8828614fc..a2d72fc9ead 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.88435, - "2": 10.88422, - "3": 10.88626, + "1": 10.88433, + "2": 10.88423, + "3": 10.88627, "4": 10.88394, - "5": 10.87627, - "6": 10.87847, + "5": 10.87625, + "6": 10.87845, "7": 10.8809, "8": 10.88013, - "9": 10.88931, - "10": 10.88048, - "11": 10.88623, - "12": 10.87876, + "9": 10.88934, + "10": 10.88046, + "11": 10.88621, + "12": 10.87877, "13": 10.89449, - "14": 10.88389, + "14": 10.88391, "15": 10.88211, - "16": 10.87773, - "17": 10.8927, - "18": 10.88925, - "19": 10.89032, - "20": 10.85878, - "21": 10.88453, + "16": 10.87774, + "17": 10.89272, + "18": 10.88924, + "19": 10.89033, + "20": 10.85877, + "21": 10.88449, "22": 10.88544, "23": 10.87545, - "24": 10.8616, - "25": 10.85065, - "26": 10.85885, + "24": 10.86159, + "25": 10.85066, + "26": 10.85883, "27": 10.83787, - "28": 10.83601, - "29": 10.81343, - "30": 10.80922, - "31": 10.78872, - "32": 10.77183, - "33": 10.7608, - "34": 10.72177, - "35": 10.71934, - "36": 10.69597, - "37": 10.67424, - "38": 10.67125, - "39": 10.63891, - "40": 10.62456, + "28": 10.83604, + "29": 10.81342, + "30": 10.80925, + "31": 10.7887, + "32": 10.77182, + "33": 10.76081, + "34": 10.72176, + "35": 10.71932, + "36": 10.69599, + "37": 10.67421, + "38": 10.67122, + "39": 10.63889, + "40": 10.62457, "41": 10.60483, - "42": 10.57112, - "43": 10.56236, - "44": 10.52105, - "45": 10.53076, - "46": 10.49086, - "47": 10.4773, - "48": 10.43282, + "42": 10.57113, + "43": 10.56235, + "44": 10.52103, + "45": 10.53074, + "46": 10.49084, + "47": 10.47728, + "48": 10.43281, "49": 10.42618, - "50": 10.4102, - "51": 10.40488, - "52": 10.36059, - "53": 10.3683, - "54": 10.32922, - "55": 10.29179, - "56": 10.30654, - "57": 10.30151, - "58": 10.29991, - "59": 10.24296, + "50": 10.41019, + "51": 10.40485, + "52": 10.36057, + "53": 10.36828, + "54": 10.3292, + "55": 10.29178, + "56": 10.30655, + "57": 10.3015, + "58": 10.29993, + "59": 10.24298, "60": 10.24907, - "61": 10.19765, - "62": 10.16495, - "63": 10.21765, - "64": 10.1818, + "61": 10.19764, + "62": 10.16494, + "63": 10.21761, + "64": 10.18178, "65": 10.15824, - "66": 10.16584, + "66": 10.16583, "67": 10.14083, - "68": 10.09888, - "69": 10.12465, + "68": 10.09887, + "69": 10.12461, "70": 10.0987, - "71": 10.11141, - "72": 10.10096, - "73": 10.08582, - "74": 10.07172, + "71": 10.11139, + "72": 10.10095, + "73": 10.08581, + "74": 10.07171, "75": 10.0404, - "76": 10.06865, + "76": 10.06863, "77": 10.06473, - "78": 10.01983, - "79": 10.02847, + "78": 10.01982, + "79": 10.02845, "80": 10.04766, "81": 10.05874, "82": 9.99574, - "83": 9.97928, - "84": 9.90854, - "85": 9.91061, - "86": 9.99801, - "87": 10.00773, + "83": 9.97927, + "84": 9.90852, + "85": 9.91059, + "86": 9.99799, + "87": 10.00771, "88": 9.99367, - "89": 9.92501, - "90": 9.91975, + "89": 9.925, + "90": 9.91973, "91": 9.95445, - "92": 9.92826, - "93": 9.86432, - "94": 9.9401, - "95": 9.93161, - "96": 9.90475, - "97": 9.85604, + "92": 9.92824, + "93": 9.86431, + "94": 9.94007, + "95": 9.9316, + "96": 9.90473, + "97": 9.85602, "98": 9.87743, - "99": 9.91981, - "100": 9.82624 + "99": 9.9198, + "100": 9.82623 } }, "mem-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.82852, - "3": 0.08859, - "4": 0.07529, - "5": 0.0756, - "6": 0.07509, - "7": 0.07455, - "8": 0.07543, - "9": 0.07492, - "10": 0.07485, - "11": 0.07447, - "12": 0.07496, - "13": 0.07485, - "14": 0.0753, - "15": 0.07566, - "16": 0.07435, - "17": 0.14289, - "18": 0.08717, - "19": 0.0853, - "20": 0.08511, - "21": 0.0867, - "22": 0.08493, - "23": 0.08511, - "24": 0.0852, - "25": 0.08498, - "26": 0.08493, - "27": 0.08476, - "28": 0.08492, - "29": 0.08486, - "30": 0.08471, - "31": 0.08493, - "32": 0.08507, - "33": 0.08463, - "34": 0.08544, - "35": 0.08496, - "36": 0.08519, - "37": 0.08545, - "38": 0.08493, - "39": 0.08494, - "40": 0.08514, - "41": 0.08522, - "42": 0.0847, - "43": 0.08472, - "44": 0.08515, - "45": 0.08463, - "46": 0.08493, - "47": 0.08518, - "48": 0.08478, - "49": 0.08493, - "50": 0.08491, - "51": 0.22226, - "52": 0.10594, - "53": 0.13076, - "54": 0.0896, - "55": 0.08433, - "56": 0.08411, - "57": 0.08397, - "58": 0.08404, - "59": 0.0838, - "60": 0.08381, - "61": 0.08402, - "62": 0.08417, - "63": 0.08405, - "64": 0.08354, - "65": 0.08416, - "66": 0.08373, - "67": 0.08433, - "68": 0.08438, - "69": 0.08443, - "70": 0.08476, - "71": 0.08454, - "72": 0.08436, - "73": 0.08396, - "74": 0.08407, - "75": 0.08456, - "76": 0.08455, - "77": 0.08422, - "78": 0.08411, - "79": 0.08534, - "80": 0.0844, - "81": 0.08461, - "82": 0.08463, - "83": 0.0847, - "84": 0.08421, - "85": 0.08445, - "86": 0.08432, - "87": 0.08453, - "88": 0.08448, - "89": 0.08462, - "90": 0.08465, - "91": 0.08462, - "92": 0.08474, - "93": 0.08411, - "94": 0.08426, - "95": 0.08373, - "96": 0.08375, - "97": 0.08411, - "98": 0.08344, - "99": 0.08402, - "100": 0.08393 + "2": 5.22172, + "3": 0.11019, + "4": 0.06414, + "5": 0.06527, + "6": 0.06452, + "7": 0.0646, + "8": 0.06493, + "9": 0.06484, + "10": 0.06431, + "11": 0.0646, + "12": 0.06488, + "13": 0.06498, + "14": 0.06483, + "15": 0.06502, + "16": 0.06525, + "17": 0.11327, + "18": 0.07643, + "19": 0.07257, + "20": 0.07248, + "21": 0.07263, + "22": 0.07222, + "23": 0.07246, + "24": 0.07159, + "25": 0.07235, + "26": 0.072, + "27": 0.07256, + "28": 0.07308, + "29": 0.0733, + "30": 0.07331, + "31": 0.07182, + "32": 0.07269, + "33": 0.07366, + "34": 0.07259, + "35": 0.07268, + "36": 0.07259, + "37": 0.0733, + "38": 0.07363, + "39": 0.07235, + "40": 0.07228, + "41": 0.07246, + "42": 0.07236, + "43": 0.07483, + "44": 0.07254, + "45": 0.07264, + "46": 0.07233, + "47": 0.07294, + "48": 0.07253, + "49": 0.07282, + "50": 0.07256, + "51": 0.24402, + "52": 0.8629, + "53": 0.07408, + "54": 0.07408, + "55": 0.0735, + "56": 0.0729, + "57": 0.07239, + "58": 0.07238, + "59": 0.07236, + "60": 0.0724, + "61": 0.07177, + "62": 0.0715, + "63": 0.07215, + "64": 0.07226, + "65": 0.07211, + "66": 0.07228, + "67": 0.07234, + "68": 0.07226, + "69": 0.07288, + "70": 0.0718, + "71": 0.07169, + "72": 0.07274, + "73": 0.07347, + "74": 0.07348, + "75": 0.07272, + "76": 0.07195, + "77": 0.07294, + "78": 0.07216, + "79": 0.07277, + "80": 0.07215, + "81": 0.07249, + "82": 0.07211, + "83": 0.07234, + "84": 0.07197, + "85": 0.07244, + "86": 0.07225, + "87": 0.07238, + "88": 0.07257, + "89": 0.0735, + "90": 0.07358, + "91": 0.07256, + "92": 0.07281, + "93": 0.0734, + "94": 0.0727, + "95": 0.07279, + "96": 0.07285, + "97": 0.07295, + "98": 0.07235, + "99": 0.07265, + "100": 0.07269 } }, "num-zeros": { @@ -448,90 +448,90 @@ "14": "nan", "15": "nan", "16": "nan", - "17": 1299.0, - "18": 1430.0, - "19": 1162.0, - "20": 1189.0, - "21": 1168.0, - "22": 1155.0, - "23": 1278.0, - "24": 1412.0, - "25": 1177.0, - "26": 1255.0, - "27": 1275.0, - "28": 1235.0, - "29": 1232.0, - "30": 1180.0, - "31": 1366.0, - "32": 1251.0, - "33": 1190.0, - "34": 1325.0, - "35": 1463.0, - "36": 1189.0, - "37": 1297.0, - "38": 1220.0, - "39": 1339.0, - "40": 1386.0, - "41": 1336.0, - "42": 1180.0, - "43": 1370.0, - "44": 1253.0, - "45": 1476.0, - "46": 1418.0, - "47": 1370.0, - "48": 1545.0, - "49": 1435.0, - "50": 1536.0, - "51": 1432.0, - "52": 1499.0, - "53": 1531.0, - "54": 1465.0, - "55": 1382.0, - "56": 1523.0, - "57": 1425.0, - "58": 1814.0, - "59": 1523.0, - "60": 1578.0, - "61": 1492.0, - "62": 1639.0, - "63": 1677.0, - "64": 1920.0, - "65": 1812.0, - "66": 1700.0, - "67": 2006.0, - "68": 2121.0, - "69": 1962.0, - "70": 1855.0, - "71": 1966.0, - "72": 1831.0, - "73": 2058.0, - "74": 2248.0, - "75": 2423.0, - "76": 2236.0, - "77": 2326.0, - "78": 2298.0, - "79": 2776.0, - "80": 2230.0, - "81": 2519.0, - "82": 1727.0, - "83": 2265.0, - "84": 2095.0, - "85": 2267.0, - "86": 2207.0, - "87": 1536.0, - "88": 2423.0, - "89": 2664.0, - "90": 2617.0, - "91": 1900.0, - "92": 2118.0, - "93": 2310.0, - "94": 1835.0, - "95": 2167.0, - "96": 2683.0, - "97": 2509.0, - "98": 2524.0, - "99": 2010.0, - "100": 2030.0 + "17": 1282.0, + "18": 1371.0, + "19": 1146.0, + "20": 1147.0, + "21": 1203.0, + "22": 1140.0, + "23": 1192.0, + "24": 1338.0, + "25": 1158.0, + "26": 1212.0, + "27": 1184.0, + "28": 1224.0, + "29": 1240.0, + "30": 1162.0, + "31": 1397.0, + "32": 1200.0, + "33": 1231.0, + "34": 1291.0, + "35": 1444.0, + "36": 1151.0, + "37": 1249.0, + "38": 1263.0, + "39": 1300.0, + "40": 1377.0, + "41": 1339.0, + "42": 1207.0, + "43": 1373.0, + "44": 1329.0, + "45": 1520.0, + "46": 1375.0, + "47": 1411.0, + "48": 1539.0, + "49": 1515.0, + "50": 1503.0, + "51": 1411.0, + "52": 1479.0, + "53": 1468.0, + "54": 1424.0, + "55": 1396.0, + "56": 1425.0, + "57": 1389.0, + "58": 1829.0, + "59": 1545.0, + "60": 1560.0, + "61": 1479.0, + "62": 1683.0, + "63": 1735.0, + "64": 1954.0, + "65": 1784.0, + "66": 1591.0, + "67": 1964.0, + "68": 2035.0, + "69": 1933.0, + "70": 2010.0, + "71": 1919.0, + "72": 1784.0, + "73": 2099.0, + "74": 2326.0, + "75": 2451.0, + "76": 2245.0, + "77": 2273.0, + "78": 2334.0, + "79": 2707.0, + "80": 2302.0, + "81": 2589.0, + "82": 1837.0, + "83": 2337.0, + "84": 2031.0, + "85": 2218.0, + "86": 2255.0, + "87": 1602.0, + "88": 2255.0, + "89": 2680.0, + "90": 2704.0, + "91": 1953.0, + "92": 2043.0, + "93": 2305.0, + "94": 1774.0, + "95": 2187.0, + "96": 2796.0, + "97": 2419.0, + "98": 2506.0, + "99": 2023.0, + "100": 1957.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json index 8123a43dcaf..e9bffe4920d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json @@ -4,105 +4,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93403, - "2": 10.92484, + "1": 10.93401, + "2": 10.92485, "3": 10.92778, - "4": 10.93122, - "5": 10.93818, - "6": 10.93103, - "7": 10.92431, - "8": 10.92541, - "9": 10.92857, - "10": 10.93338, - "11": 10.92137, - "12": 10.94242, - "13": 10.91106, - "14": 10.913, - "15": 10.93447, - "16": 10.92125, - "17": 10.91791, - "18": 10.93125, - "19": 10.91067, - "20": 10.93038, + "4": 10.93123, + "5": 10.9382, + "6": 10.93105, + "7": 10.9243, + "8": 10.9254, + "9": 10.92859, + "10": 10.93342, + "11": 10.92141, + "12": 10.94246, + "13": 10.91109, + "14": 10.91299, + "15": 10.93443, + "16": 10.92127, + "17": 10.91789, + "18": 10.93123, + "19": 10.91064, + "20": 10.93035, "21": 10.92367, "22": 10.92127, - "23": 10.93319, - "24": 10.92834, - "25": 10.9237, - "26": 10.90616, - "27": 10.90542, - "28": 10.87556, - "29": 10.86744, - "30": 10.85414, - "31": 10.8473, - "32": 10.83632, + "23": 10.93316, + "24": 10.92838, + "25": 10.92371, + "26": 10.90611, + "27": 10.90541, + "28": 10.87557, + "29": 10.86748, + "30": 10.85411, + "31": 10.84729, + "32": 10.83631, "33": 10.8269, - "34": 10.79439, - "35": 10.79214, - "36": 10.77173, - "37": 10.73975, + "34": 10.79435, + "35": 10.79215, + "36": 10.77174, + "37": 10.73974, "38": 10.72731, - "39": 10.70624, - "40": 10.67793, + "39": 10.70622, + "40": 10.67794, "41": 10.65748, - "42": 10.65898, - "43": 10.61991, + "42": 10.659, + "43": 10.61993, "44": 10.60924, - "45": 10.59429, + "45": 10.59428, "46": 10.56291, "47": 10.5467, - "48": 10.51856, - "49": 10.49105, - "50": 10.48832, + "48": 10.51851, + "49": 10.49103, + "50": 10.48829, "51": 10.45968, - "52": 10.42657, + "52": 10.42655, "53": 10.42446, - "54": 10.38679, + "54": 10.3868, "55": 10.36297, "56": 10.36284, - "57": 10.34462, - "58": 10.33908, + "57": 10.34458, + "58": 10.33907, "59": 10.29223, - "60": 10.30273, - "61": 10.25456, + "60": 10.3027, + "61": 10.25457, "62": 10.22031, - "63": 10.26035, - "64": 10.21252, + "63": 10.26031, + "64": 10.21251, "65": 10.18633, - "66": 10.21017, - "67": 10.17978, - "68": 10.15173, - "69": 10.15752, - "70": 10.12972, - "71": 10.14275, - "72": 10.13563, - "73": 10.11602, - "74": 10.10391, - "75": 10.06842, + "66": 10.21015, + "67": 10.17977, + "68": 10.15174, + "69": 10.15751, + "70": 10.12971, + "71": 10.14273, + "72": 10.13562, + "73": 10.11601, + "74": 10.1039, + "75": 10.06841, "76": 10.08526, "77": 10.08504, - "78": 10.0442, - "79": 10.04401, + "78": 10.04419, + "79": 10.04399, "80": 10.06071, - "81": 10.07915, - "82": 10.01324, + "81": 10.07914, + "82": 10.01325, "83": 9.99482, - "84": 9.93195, - "85": 9.92598, - "86": 10.01583, - "87": 10.0155, - "88": 10.0004, - "89": 9.93966, + "84": 9.93194, + "85": 9.92597, + "86": 10.01581, + "87": 10.01549, + "88": 10.00038, + "89": 9.93965, "90": 9.93958, - "91": 9.96523, - "92": 9.94256, + "91": 9.96522, + "92": 9.94253, "93": 9.87628, "94": 9.95234, "95": 9.93593, - "96": 9.91366, - "97": 9.87406, - "98": 9.88465, - "99": 9.93057, + "96": 9.91363, + "97": 9.87403, + "98": 9.88464, + "99": 9.93054, "100": 9.82911 } }, @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.78632, - "3": 0.06568, - "4": 0.04825, - "5": 0.04726, - "6": 0.04827, - "7": 0.0471, - "8": 0.04819, - "9": 0.04844, - "10": 0.04813, - "11": 0.04818, - "12": 0.048, - "13": 0.04993, - "14": 0.0472, - "15": 0.04828, - "16": 0.0481, - "17": 0.08931, - "18": 0.05484, - "19": 0.05166, - "20": 0.05057, - "21": 0.05154, - "22": 0.05245, - "23": 0.05167, - "24": 0.05294, - "25": 0.05229, - "26": 0.05253, - "27": 0.05178, - "28": 0.05234, - "29": 0.0521, - "30": 0.05222, - "31": 0.05162, - "32": 0.05252, - "33": 0.05174, - "34": 0.05215, - "35": 0.052, - "36": 0.05212, - "37": 0.05157, - "38": 0.05247, - "39": 0.05189, - "40": 0.05239, - "41": 0.05136, - "42": 0.05238, - "43": 0.05183, - "44": 0.05217, - "45": 0.05148, - "46": 0.05156, - "47": 0.05137, - "48": 0.05224, - "49": 0.05175, - "50": 0.05215, - "51": 0.12023, - "52": 0.05543, - "53": 0.09332, - "54": 0.05491, - "55": 0.05535, - "56": 0.05419, - "57": 0.05423, - "58": 0.05292, - "59": 0.05271, - "60": 0.0524, - "61": 0.05324, - "62": 0.05206, - "63": 0.05227, - "64": 0.05227, - "65": 0.05427, - "66": 0.05224, - "67": 0.05223, - "68": 0.05212, - "69": 0.05212, - "70": 0.05377, - "71": 0.05285, - "72": 0.05268, - "73": 0.05261, - "74": 0.05252, - "75": 0.05347, - "76": 0.05297, - "77": 0.05306, - "78": 0.05308, - "79": 0.05274, - "80": 0.05266, - "81": 0.05235, - "82": 0.05315, - "83": 0.05148, - "84": 0.05377, - "85": 0.05238, - "86": 0.05192, - "87": 0.05345, - "88": 0.05239, - "89": 0.05137, - "90": 0.05224, - "91": 0.05185, - "92": 0.05214, - "93": 0.05102, - "94": 0.05437, - "95": 0.05134, - "96": 0.05215, - "97": 0.05242, - "98": 0.05178, - "99": 0.05181, - "100": 0.05586 + "2": 3.34299, + "3": 0.06965, + "4": 0.04839, + "5": 0.05031, + "6": 0.04913, + "7": 0.05028, + "8": 0.04826, + "9": 0.05045, + "10": 0.0486, + "11": 0.05024, + "12": 0.04877, + "13": 0.05031, + "14": 0.04906, + "15": 0.05012, + "16": 0.04901, + "17": 0.0914, + "18": 0.05578, + "19": 0.05789, + "20": 0.051, + "21": 0.05523, + "22": 0.05393, + "23": 0.05479, + "24": 0.05343, + "25": 0.05456, + "26": 0.05369, + "27": 0.05438, + "28": 0.05321, + "29": 0.05477, + "30": 0.05282, + "31": 0.05486, + "32": 0.05328, + "33": 0.05492, + "34": 0.05356, + "35": 0.05482, + "36": 0.05321, + "37": 0.05467, + "38": 0.0531, + "39": 0.05491, + "40": 0.05447, + "41": 0.05529, + "42": 0.0536, + "43": 0.05502, + "44": 0.05302, + "45": 0.05482, + "46": 0.05289, + "47": 0.05485, + "48": 0.05275, + "49": 0.05484, + "50": 0.05503, + "51": 0.11526, + "52": 0.0693, + "53": 0.80406, + "54": 0.05828, + "55": 0.05456, + "56": 0.05458, + "57": 0.0547, + "58": 0.05445, + "59": 0.05427, + "60": 0.05476, + "61": 0.05404, + "62": 0.05462, + "63": 0.05439, + "64": 0.05455, + "65": 0.05434, + "66": 0.05456, + "67": 0.05456, + "68": 0.0568, + "69": 0.05468, + "70": 0.05554, + "71": 0.05447, + "72": 0.05443, + "73": 0.05444, + "74": 0.05447, + "75": 0.05389, + "76": 0.05468, + "77": 0.05417, + "78": 0.05468, + "79": 0.05417, + "80": 0.05462, + "81": 0.05427, + "82": 0.05507, + "83": 0.05423, + "84": 0.05477, + "85": 0.05431, + "86": 0.05737, + "87": 0.05566, + "88": 0.05437, + "89": 0.05441, + "90": 0.0544, + "91": 0.05575, + "92": 0.05439, + "93": 0.05532, + "94": 0.05465, + "95": 0.05551, + "96": 0.05446, + "97": 0.05561, + "98": 0.05454, + "99": 0.05585, + "100": 0.05478 } }, "num-zeros": { @@ -448,90 +448,90 @@ "14": "nan", "15": "nan", "16": "nan", - "17": 1230.0, - "18": 1439.0, - "19": 1177.0, + "17": 1224.0, + "18": 1326.0, + "19": 1192.0, "20": "nan", - "21": 1304.0, - "22": 1240.0, - "23": 1239.0, - "24": 1401.0, - "25": 1151.0, - "26": 1247.0, - "27": 1256.0, - "28": 1240.0, - "29": 1313.0, - "30": 1261.0, - "31": 1401.0, - "32": 1230.0, - "33": 1182.0, - "34": 1324.0, - "35": 1544.0, - "36": 1254.0, - "37": 1252.0, - "38": 1243.0, - "39": 1220.0, - "40": 1374.0, - "41": 1301.0, - "42": 1167.0, - "43": 1412.0, - "44": 1238.0, - "45": 1490.0, - "46": 1416.0, - "47": 1399.0, - "48": 1495.0, - "49": 1424.0, - "50": 1492.0, - "51": 1297.0, - "52": 1460.0, - "53": 1449.0, - "54": 1434.0, - "55": 1302.0, - "56": 1448.0, - "57": 1424.0, - "58": 1752.0, - "59": 1510.0, - "60": 1550.0, - "61": 1407.0, - "62": 1625.0, - "63": 1631.0, - "64": 1751.0, - "65": 1720.0, - "66": 1596.0, - "67": 1845.0, - "68": 1992.0, - "69": 1947.0, - "70": 1783.0, - "71": 1840.0, - "72": 1785.0, - "73": 1957.0, - "74": 2177.0, - "75": 2191.0, - "76": 2207.0, - "77": 2364.0, - "78": 2242.0, - "79": 2500.0, - "80": 2301.0, - "81": 2551.0, - "82": 1712.0, - "83": 2201.0, - "84": 2138.0, - "85": 2133.0, - "86": 2133.0, - "87": 1651.0, - "88": 2355.0, - "89": 2589.0, - "90": 2628.0, - "91": 1867.0, - "92": 2185.0, - "93": 2383.0, - "94": 1779.0, - "95": 2042.0, - "96": 2630.0, - "97": 2298.0, - "98": 2356.0, - "99": 2059.0, - "100": 2045.0 + "21": 1250.0, + "22": 1156.0, + "23": 1183.0, + "24": 1519.0, + "25": 1251.0, + "26": 1301.0, + "27": 1278.0, + "28": 1281.0, + "29": 1296.0, + "30": 1252.0, + "31": 1379.0, + "32": 1174.0, + "33": 1181.0, + "34": 1343.0, + "35": 1487.0, + "36": 1177.0, + "37": 1268.0, + "38": 1206.0, + "39": 1251.0, + "40": 1341.0, + "41": 1424.0, + "42": 1151.0, + "43": 1409.0, + "44": 1267.0, + "45": 1513.0, + "46": 1340.0, + "47": 1361.0, + "48": 1435.0, + "49": 1436.0, + "50": 1479.0, + "51": 1425.0, + "52": 1544.0, + "53": 1561.0, + "54": 1454.0, + "55": 1351.0, + "56": 1484.0, + "57": 1361.0, + "58": 1798.0, + "59": 1424.0, + "60": 1533.0, + "61": 1473.0, + "62": 1636.0, + "63": 1647.0, + "64": 1856.0, + "65": 1767.0, + "66": 1576.0, + "67": 1801.0, + "68": 2018.0, + "69": 2005.0, + "70": 1811.0, + "71": 1905.0, + "72": 1954.0, + "73": 1936.0, + "74": 2133.0, + "75": 2273.0, + "76": 2224.0, + "77": 2201.0, + "78": 2268.0, + "79": 2630.0, + "80": 2392.0, + "81": 2511.0, + "82": 1843.0, + "83": 2228.0, + "84": 2089.0, + "85": 2228.0, + "86": 2181.0, + "87": 1724.0, + "88": 2322.0, + "89": 2633.0, + "90": 2595.0, + "91": 1870.0, + "92": 2158.0, + "93": 2342.0, + "94": 1776.0, + "95": 2079.0, + "96": 2729.0, + "97": 2347.0, + "98": 2364.0, + "99": 2013.0, + "100": 2122.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json index 4e025fbf4ed..51e48126885 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json @@ -4,55 +4,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.90498, - "2": 10.91139, - "3": 10.91306, - "4": 10.91044, - "5": 10.90317, - "6": 10.88591, - "7": 10.89792, - "8": 10.90656, + "1": 10.90496, + "2": 10.91137, + "3": 10.91308, + "4": 10.9104, + "5": 10.90318, + "6": 10.88585, + "7": 10.89794, + "8": 10.90651, "9": 10.90486, - "10": 10.90466, - "11": 10.88737, - "12": 10.88996, - "13": 10.87962, - "14": 10.87793, - "15": 10.86811, - "16": 10.85418, - "17": 10.85643, - "18": 10.84095, - "19": 10.84003, - "20": 10.75474, - "21": 10.75943, - "22": 10.74755, - "23": 10.73819, - "24": 10.69892, - "25": 10.71401, - "26": 10.68879, - "27": 10.65358, - "28": 10.57721, + "10": 10.90463, + "11": 10.88745, + "12": 10.89004, + "13": 10.87956, + "14": 10.87798, + "15": 10.86819, + "16": 10.85414, + "17": 10.85642, + "18": 10.84094, + "19": 10.84004, + "20": 10.75481, + "21": 10.75946, + "22": 10.74768, + "23": 10.73823, + "24": 10.69888, + "25": 10.71402, + "26": 10.68881, + "27": 10.65359, + "28": 10.57728, "29": 10.55724, "30": 10.53011, - "31": 10.52624, - "32": 10.50557, - "33": 10.48143, - "34": 10.43308, - "35": 10.44066, - "36": 10.42283, - "37": 10.37607, - "38": 10.38561, - "39": 10.34763, - "40": 10.34018, - "41": 10.30835, - "42": 10.29875, - "43": 10.27584, - "44": 10.24481, - "45": 10.25951, + "31": 10.52632, + "32": 10.50564, + "33": 10.48147, + "34": 10.43316, + "35": 10.44071, + "36": 10.42287, + "37": 10.37614, + "38": 10.38565, + "39": 10.34767, + "40": 10.34024, + "41": 10.30838, + "42": 10.2988, + "43": 10.27593, + "44": 10.24485, + "45": 10.25957, "46": 10.22712, - "47": 10.21219, - "48": 10.16164, - "49": 10.16138, + "47": 10.21217, + "48": 10.16162, + "49": 10.16146, "50": 10.16839 } }, @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1736.0, - "2": 1711.0, - "3": 1770.0, - "4": 1666.0, - "5": 1709.0, - "6": 1729.0, - "7": 1877.0, - "8": 1645.0, - "9": 1711.0, - "10": 1777.0, - "11": 1645.0, - "12": 1716.0, - "13": 1866.0, - "14": 1783.0, - "15": 1572.0, - "16": 1772.0, - "17": 1772.0, - "18": 1693.0, - "19": 1677.0, - "20": 1624.0, - "21": 1735.0, - "22": 1728.0, - "23": 1787.0, - "24": 1977.0, - "25": 1690.0, - "26": 1791.0, - "27": 1782.0, - "28": 1746.0, - "29": 1922.0, - "30": 1679.0, - "31": 1897.0, - "32": 1943.0, - "33": 1987.0, - "34": 2042.0, + "1": 1723.0, + "2": 1661.0, + "3": 1732.0, + "4": 1671.0, + "5": 1695.0, + "6": 1757.0, + "7": 1821.0, + "8": 1765.0, + "9": 1852.0, + "10": 1764.0, + "11": 1714.0, + "12": 1704.0, + "13": 1814.0, + "14": 1799.0, + "15": 1567.0, + "16": 1720.0, + "17": 1660.0, + "18": 1778.0, + "19": 1694.0, + "20": 1692.0, + "21": 1811.0, + "22": 1717.0, + "23": 1737.0, + "24": 1821.0, + "25": 1822.0, + "26": 1785.0, + "27": 1807.0, + "28": 1777.0, + "29": 1933.0, + "30": 1748.0, + "31": 1958.0, + "32": 1916.0, + "33": 1970.0, + "34": 2136.0, "35": 2033.0, - "36": 1981.0, - "37": 2180.0, - "38": 2196.0, - "39": 2236.0, - "40": 2183.0, - "41": 2259.0, - "42": 2000.0, - "43": 2417.0, - "44": 2183.0, - "45": 2591.0, - "46": 2486.0, - "47": 2376.0, - "48": 2614.0, - "49": 2856.0, - "50": 2626.0 + "36": 1979.0, + "37": 2288.0, + "38": 2042.0, + "39": 2134.0, + "40": 2246.0, + "41": 2306.0, + "42": 1986.0, + "43": 2425.0, + "44": 2206.0, + "45": 2640.0, + "46": 2535.0, + "47": 2492.0, + "48": 2554.0, + "49": 2869.0, + "50": 2525.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.33315, - "3": 0.14175, - "4": 0.13031, - "5": 0.12954, - "6": 0.12904, - "7": 0.12895, - "8": 0.12777, - "9": 0.1283, - "10": 0.12927, - "11": 0.12911, - "12": 0.1289, - "13": 0.12884, - "14": 0.12931, - "15": 0.12945, - "16": 0.12954, - "17": 0.12947, - "18": 0.13099, - "19": 0.12968, - "20": 0.13066, - "21": 0.13075, - "22": 0.13015, - "23": 0.13027, - "24": 0.12985, - "25": 0.13, - "26": 0.1292, - "27": 0.13034, - "28": 0.1303, - "29": 0.13085, - "30": 0.13022, - "31": 0.13028, - "32": 0.12968, - "33": 0.12906, - "34": 0.12946, - "35": 0.1297, - "36": 0.12914, - "37": 0.12955, - "38": 0.12974, - "39": 0.12988, - "40": 0.12956, - "41": 0.12929, - "42": 0.12916, - "43": 0.12958, - "44": 0.12938, - "45": 0.12917, - "46": 0.12972, - "47": 0.12889, - "48": 0.13257, - "49": 0.12937, - "50": 0.12951 + "2": 7.067, + "3": 0.16652, + "4": 0.10704, + "5": 0.10752, + "6": 0.10747, + "7": 0.10686, + "8": 0.10679, + "9": 0.10793, + "10": 0.10755, + "11": 0.10708, + "12": 0.10734, + "13": 0.10783, + "14": 0.10793, + "15": 0.10758, + "16": 0.10682, + "17": 0.10794, + "18": 0.10816, + "19": 0.10759, + "20": 0.1068, + "21": 0.10872, + "22": 0.10827, + "23": 0.10866, + "24": 0.10832, + "25": 0.10691, + "26": 0.10813, + "27": 0.10884, + "28": 0.10866, + "29": 0.1089, + "30": 0.10919, + "31": 0.10764, + "32": 0.10902, + "33": 0.1091, + "34": 0.10905, + "35": 0.10904, + "36": 0.10763, + "37": 0.10909, + "38": 0.10895, + "39": 0.10959, + "40": 0.10886, + "41": 0.10893, + "42": 0.10902, + "43": 0.10896, + "44": 0.10872, + "45": 0.10827, + "46": 0.10666, + "47": 0.10839, + "48": 0.10689, + "49": 0.10898, + "50": 0.10702 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_h100.json index b9694a444a0..a4f042b8734 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.9492, - "2": 10.92944, - "3": 10.92617, - "4": 10.93599, - "5": 10.94247, - "6": 10.93529, - "7": 10.92769, - "8": 10.93485, + "1": 10.94919, + "2": 10.92948, + "3": 10.92613, + "4": 10.93601, + "5": 10.94236, + "6": 10.93534, + "7": 10.9276, + "8": 10.93495, "9": 10.93174, "10": 10.9248, - "11": 10.92287, - "12": 10.93365, - "13": 10.90794, - "14": 10.90573, - "15": 10.89216, - "16": 10.87883, - "17": 10.8763, - "18": 10.87025, - "19": 10.86569, - "20": 10.78889, - "21": 10.77485, - "22": 10.75682, - "23": 10.76031, - "24": 10.72885, - "25": 10.73044, + "11": 10.92284, + "12": 10.93371, + "13": 10.90792, + "14": 10.90564, + "15": 10.89215, + "16": 10.87885, + "17": 10.87632, + "18": 10.87026, + "19": 10.86574, + "20": 10.78902, + "21": 10.77484, + "22": 10.7568, + "23": 10.76039, + "24": 10.72889, + "25": 10.73048, "26": 10.71932, - "27": 10.66854, - "28": 10.59297, - "29": 10.5737, - "30": 10.55609, - "31": 10.55354, - "32": 10.533, - "33": 10.50296, - "34": 10.46461, + "27": 10.6686, + "28": 10.59299, + "29": 10.57377, + "30": 10.55615, + "31": 10.55362, + "32": 10.53296, + "33": 10.503, + "34": 10.46468, "35": 10.47716, - "36": 10.44305, - "37": 10.41348, - "38": 10.41905, - "39": 10.38957, - "40": 10.36395, - "41": 10.34559, - "42": 10.33341, - "43": 10.29681, - "44": 10.29172, - "45": 10.28787, - "46": 10.25557, - "47": 10.24096, - "48": 10.19438, - "49": 10.20509, - "50": 10.20065 + "36": 10.44314, + "37": 10.41349, + "38": 10.41904, + "39": 10.38962, + "40": 10.36399, + "41": 10.34568, + "42": 10.3334, + "43": 10.29685, + "44": 10.29174, + "45": 10.28789, + "46": 10.25563, + "47": 10.24097, + "48": 10.19443, + "49": 10.20505, + "50": 10.20072 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1718.0, - "2": 1609.0, - "3": 1632.0, - "4": 1688.0, - "5": 1717.0, - "6": 1662.0, - "7": 1920.0, - "8": 1698.0, - "9": 1716.0, - "10": 1643.0, - "11": 1641.0, - "12": 1662.0, - "13": 1842.0, - "14": 1796.0, - "15": 1624.0, - "16": 1718.0, - "17": 1760.0, - "18": 1738.0, - "19": 1682.0, - "20": 1695.0, - "21": 1789.0, - "22": 1623.0, - "23": 1732.0, - "24": 1740.0, - "25": 1829.0, - "26": 1714.0, - "27": 1805.0, - "28": 1770.0, - "29": 1913.0, - "30": 1847.0, - "31": 1902.0, - "32": 1986.0, - "33": 1935.0, - "34": 2038.0, - "35": 2123.0, - "36": 1985.0, - "37": 2193.0, - "38": 2072.0, - "39": 2223.0, - "40": 2210.0, - "41": 2213.0, - "42": 2070.0, - "43": 2326.0, - "44": 2262.0, - "45": 2551.0, - "46": 2383.0, - "47": 2312.0, - "48": 2578.0, - "49": 2778.0, - "50": 2525.0 + "1": 1743.0, + "2": 1652.0, + "3": 1710.0, + "4": 1667.0, + "5": 1652.0, + "6": 1649.0, + "7": 1851.0, + "8": 1736.0, + "9": 1653.0, + "10": 1573.0, + "11": 1689.0, + "12": 1725.0, + "13": 1777.0, + "14": 1787.0, + "15": 1618.0, + "16": 1608.0, + "17": 1752.0, + "18": 1773.0, + "19": 1613.0, + "20": 1715.0, + "21": 1757.0, + "22": 1589.0, + "23": 1591.0, + "24": 1826.0, + "25": 1775.0, + "26": 1740.0, + "27": 1784.0, + "28": 1807.0, + "29": 1818.0, + "30": 1800.0, + "31": 2006.0, + "32": 1959.0, + "33": 2001.0, + "34": 2046.0, + "35": 2086.0, + "36": 1894.0, + "37": 2185.0, + "38": 2084.0, + "39": 2261.0, + "40": 2170.0, + "41": 2218.0, + "42": 1975.0, + "43": 2319.0, + "44": 2214.0, + "45": 2553.0, + "46": 2465.0, + "47": 2382.0, + "48": 2630.0, + "49": 2716.0, + "50": 2576.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 952847360.0, - "2": 952847360.0, - "3": 952847360.0, - "4": 952847360.0, - "5": 952847360.0, - "6": 952847360.0, - "7": 952847360.0, - "8": 952847360.0, - "9": 952847360.0, - "10": 952847360.0, - "11": 952847360.0, - "12": 952847360.0, - "13": 952847360.0, - "14": 952847360.0, - "15": 952847360.0, - "16": 952847360.0, - "17": 952847360.0, - "18": 952847360.0, - "19": 952847360.0, - "20": 952847360.0, - "21": 952847360.0, - "22": 952847360.0, - "23": 952847360.0, - "24": 952847360.0, - "25": 952847360.0, - "26": 952847360.0, - "27": 952847360.0, - "28": 952847360.0, - "29": 952847360.0, - "30": 952847360.0, - "31": 952847360.0, - "32": 952847360.0, - "33": 952847360.0, - "34": 952847360.0, - "35": 952847360.0, - "36": 952847360.0, - "37": 952847360.0, - "38": 952847360.0, - "39": 952847360.0, - "40": 952847360.0, - "41": 952847360.0, - "42": 952847360.0, - "43": 952847360.0, - "44": 952847360.0, - "45": 952847360.0, - "46": 952847360.0, - "47": 952847360.0, - "48": 952847360.0, - "49": 952847360.0, - "50": 952847360.0 + "1": 948653056.0, + "2": 948653056.0, + "3": 948653056.0, + "4": 948653056.0, + "5": 948653056.0, + "6": 948653056.0, + "7": 948653056.0, + "8": 948653056.0, + "9": 948653056.0, + "10": 948653056.0, + "11": 948653056.0, + "12": 948653056.0, + "13": 948653056.0, + "14": 948653056.0, + "15": 948653056.0, + "16": 948653056.0, + "17": 948653056.0, + "18": 948653056.0, + "19": 948653056.0, + "20": 948653056.0, + "21": 948653056.0, + "22": 948653056.0, + "23": 948653056.0, + "24": 948653056.0, + "25": 948653056.0, + "26": 948653056.0, + "27": 948653056.0, + "28": 948653056.0, + "29": 948653056.0, + "30": 948653056.0, + "31": 948653056.0, + "32": 948653056.0, + "33": 948653056.0, + "34": 948653056.0, + "35": 948653056.0, + "36": 948653056.0, + "37": 948653056.0, + "38": 948653056.0, + "39": 948653056.0, + "40": 948653056.0, + "41": 948653056.0, + "42": 948653056.0, + "43": 948653056.0, + "44": 948653056.0, + "45": 948653056.0, + "46": 948653056.0, + "47": 948653056.0, + "48": 948653056.0, + "49": 948653056.0, + "50": 948653056.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3275808768.0, - "2": 3637371904.0, - "3": 3637371904.0, - "4": 3637371904.0, - "5": 3637371904.0, - "6": 3637371904.0, - "7": 3637371904.0, - "8": 3637371904.0, - "9": 3637371904.0, - "10": 3637371904.0, - "11": 3637371904.0, - "12": 3637371904.0, - "13": 3637371904.0, - "14": 3637371904.0, - "15": 3637371904.0, - "16": 3637371904.0, - "17": 3637371904.0, - "18": 3637371904.0, - "19": 3637371904.0, - "20": 3637371904.0, - "21": 3637371904.0, - "22": 3637371904.0, - "23": 3637371904.0, - "24": 3637371904.0, - "25": 3637371904.0, - "26": 3637371904.0, - "27": 3637371904.0, - "28": 3637371904.0, - "29": 3637371904.0, - "30": 3637371904.0, - "31": 3637371904.0, - "32": 3637371904.0, - "33": 3637371904.0, - "34": 3637371904.0, - "35": 3637371904.0, - "36": 3637371904.0, - "37": 3637371904.0, - "38": 3637371904.0, - "39": 3637371904.0, - "40": 3637371904.0, - "41": 3637371904.0, - "42": 3637371904.0, - "43": 3637371904.0, - "44": 3637371904.0, - "45": 3637371904.0, - "46": 3637371904.0, - "47": 3637371904.0, - "48": 3637371904.0, - "49": 3637371904.0, - "50": 3637371904.0 + "1": 3276857344.0, + "2": 3633177600.0, + "3": 3633177600.0, + "4": 3633177600.0, + "5": 3633177600.0, + "6": 3633177600.0, + "7": 3633177600.0, + "8": 3633177600.0, + "9": 3633177600.0, + "10": 3633177600.0, + "11": 3633177600.0, + "12": 3633177600.0, + "13": 3633177600.0, + "14": 3633177600.0, + "15": 3633177600.0, + "16": 3633177600.0, + "17": 3633177600.0, + "18": 3633177600.0, + "19": 3633177600.0, + "20": 3633177600.0, + "21": 3633177600.0, + "22": 3633177600.0, + "23": 3633177600.0, + "24": 3633177600.0, + "25": 3633177600.0, + "26": 3633177600.0, + "27": 3633177600.0, + "28": 3633177600.0, + "29": 3633177600.0, + "30": 3633177600.0, + "31": 3633177600.0, + "32": 3633177600.0, + "33": 3633177600.0, + "34": 3633177600.0, + "35": 3633177600.0, + "36": 3633177600.0, + "37": 3633177600.0, + "38": 3633177600.0, + "39": 3633177600.0, + "40": 3633177600.0, + "41": 3633177600.0, + "42": 3633177600.0, + "43": 3633177600.0, + "44": 3633177600.0, + "45": 3633177600.0, + "46": 3633177600.0, + "47": 3633177600.0, + "48": 3633177600.0, + "49": 3633177600.0, + "50": 3633177600.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.00806, - "3": 0.10532, - "4": 0.08399, - "5": 0.08507, - "6": 0.08371, - "7": 0.08421, - "8": 0.08346, - "9": 0.08354, - "10": 0.08318, - "11": 0.08303, - "12": 0.08312, - "13": 0.08365, - "14": 0.08374, - "15": 0.08334, - "16": 0.08407, - "17": 0.08251, - "18": 0.08274, - "19": 0.08221, - "20": 0.08228, - "21": 0.08223, - "22": 0.08241, - "23": 0.08253, - "24": 0.08212, - "25": 0.08413, - "26": 0.08249, - "27": 0.08229, - "28": 0.08329, - "29": 0.08256, - "30": 0.08302, - "31": 0.08319, - "32": 0.08483, - "33": 0.0828, - "34": 0.08235, - "35": 0.08266, - "36": 0.08226, - "37": 0.08261, - "38": 0.0821, - "39": 0.08223, - "40": 0.08282, - "41": 0.08228, - "42": 0.08368, - "43": 0.0823, - "44": 0.08379, - "45": 0.08264, - "46": 0.08245, - "47": 0.08213, - "48": 0.08261, - "49": 0.08155, - "50": 0.08164 + "2": 4.42105, + "3": 0.10675, + "4": 0.08533, + "5": 0.08587, + "6": 0.08595, + "7": 0.08663, + "8": 0.08617, + "9": 0.08709, + "10": 0.08683, + "11": 0.08644, + "12": 0.08634, + "13": 0.08694, + "14": 0.08684, + "15": 0.08676, + "16": 0.08651, + "17": 0.08661, + "18": 0.08625, + "19": 0.08615, + "20": 0.087, + "21": 0.08733, + "22": 0.08737, + "23": 0.08775, + "24": 0.08711, + "25": 0.08663, + "26": 0.08635, + "27": 0.0863, + "28": 0.08626, + "29": 0.08632, + "30": 0.08705, + "31": 0.0863, + "32": 0.08616, + "33": 0.08656, + "34": 0.08626, + "35": 0.08633, + "36": 0.08628, + "37": 0.0871, + "38": 0.08608, + "39": 0.08658, + "40": 0.08654, + "41": 0.08846, + "42": 0.08687, + "43": 0.08613, + "44": 0.0865, + "45": 0.0863, + "46": 0.08648, + "47": 0.08619, + "48": 0.08609, + "49": 0.08714, + "50": 0.08619 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_a100.json index bb5b44b45ab..88c2491dc39 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_a100.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93235, - "2": 10.93461, - "3": 10.93175, - "4": 10.93461, - "5": 10.93653, - "6": 10.9322, + "2": 10.93463, + "3": 10.93176, + "4": 10.93463, + "5": 10.93651, + "6": 10.93217, "7": 10.93579, "8": 10.93151, - "9": 10.93947, - "10": 10.93193, + "9": 10.93944, + "10": 10.93194, "11": 10.92682, - "12": 10.92752, - "13": 10.90888, - "14": 10.92629, - "15": 10.92949, - "16": 10.92729, - "17": 10.9233, - "18": 10.93297, - "19": 10.92607, - "20": 10.91811, - "21": 10.92561, - "22": 10.92733, - "23": 10.93056, - "24": 10.91254, - "25": 10.91929, - "26": 10.89481, - "27": 10.88597, - "28": 10.86044, - "29": 10.85687, - "30": 10.85957, - "31": 10.83466, - "32": 10.82409, - "33": 10.81178, - "34": 10.76637, - "35": 10.76752, - "36": 10.75748, - "37": 10.73514, - "38": 10.70881, + "12": 10.92751, + "13": 10.90892, + "14": 10.9263, + "15": 10.92947, + "16": 10.92728, + "17": 10.92331, + "18": 10.933, + "19": 10.92605, + "20": 10.91812, + "21": 10.9256, + "22": 10.92735, + "23": 10.93058, + "24": 10.91252, + "25": 10.91926, + "26": 10.89476, + "27": 10.88601, + "28": 10.86039, + "29": 10.85685, + "30": 10.85956, + "31": 10.83464, + "32": 10.82405, + "33": 10.81176, + "34": 10.76634, + "35": 10.76749, + "36": 10.75749, + "37": 10.73516, + "38": 10.70878, "39": 10.69118, - "40": 10.66846, - "41": 10.64152, + "40": 10.66844, + "41": 10.64149, "42": 10.62148, - "43": 10.60065, + "43": 10.60062, "44": 10.57683, "45": 10.57174, - "46": 10.53397, - "47": 10.50881, - "48": 10.47796, - "49": 10.45018, - "50": 10.4463, - "51": 10.43674, + "46": 10.53396, + "47": 10.50882, + "48": 10.47792, + "49": 10.45015, + "50": 10.44625, + "51": 10.43672, "52": 10.3923, - "53": 10.38086, - "54": 10.35609, - "55": 10.31685, - "56": 10.31928, - "57": 10.29912, - "58": 10.29984, - "59": 10.24223, - "60": 10.26102, - "61": 10.20428, - "62": 10.17332, - "63": 10.22247, - "64": 10.16981, - "65": 10.14463, + "53": 10.38085, + "54": 10.35608, + "55": 10.31681, + "56": 10.31929, + "57": 10.2991, + "58": 10.29986, + "59": 10.24221, + "60": 10.261, + "61": 10.20426, + "62": 10.17333, + "63": 10.22243, + "64": 10.16978, + "65": 10.14461, "66": 10.15709, "67": 10.13188, - "68": 10.09274, - "69": 10.11764, - "70": 10.08312, - "71": 10.10337, - "72": 10.08268, - "73": 10.07747, - "74": 10.06098, - "75": 10.03279, - "76": 10.04681, - "77": 10.05183, + "68": 10.09271, + "69": 10.11763, + "70": 10.08313, + "71": 10.10336, + "72": 10.08267, + "73": 10.07746, + "74": 10.06096, + "75": 10.03277, + "76": 10.0468, + "77": 10.05181, "78": 10.01472, - "79": 10.01713, - "80": 10.03438, - "81": 10.05145, - "82": 9.98403, + "79": 10.01712, + "80": 10.03437, + "81": 10.05146, + "82": 9.98401, "83": 9.97304, "84": 9.9166, - "85": 9.89447, - "86": 9.99429, - "87": 9.99851, - "88": 9.97896, - "89": 9.91738, - "90": 9.91083, - "91": 9.9378, - "92": 9.92283, - "93": 9.86296, - "94": 9.93396, - "95": 9.92146, - "96": 9.91104, - "97": 9.86072, - "98": 9.87274, - "99": 9.91227, - "100": 9.81787 + "85": 9.89446, + "86": 9.99428, + "87": 9.9985, + "88": 9.97895, + "89": 9.91739, + "90": 9.91082, + "91": 9.93778, + "92": 9.92281, + "93": 9.86295, + "94": 9.93394, + "95": 9.92144, + "96": 9.91101, + "97": 9.86071, + "98": 9.87273, + "99": 9.91225, + "100": 9.81785 } }, "mem-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.97766, - "3": 0.1287, - "4": 0.10583, - "5": 0.1069, - "6": 0.10618, - "7": 0.10574, - "8": 0.10612, - "9": 0.10591, - "10": 0.10698, - "11": 0.10693, - "12": 0.10663, - "13": 0.10585, - "14": 0.10615, - "15": 0.1065, - "16": 0.1794, - "17": 0.1243, - "18": 0.10663, - "19": 0.11383, - "20": 0.11407, - "21": 0.11377, - "22": 0.11379, - "23": 0.11418, - "24": 0.11372, - "25": 0.11476, - "26": 0.11383, - "27": 0.11454, - "28": 0.11405, - "29": 0.11459, - "30": 0.11416, - "31": 0.11441, - "32": 0.11474, - "33": 0.11493, - "34": 0.11382, - "35": 0.11438, - "36": 0.11491, - "37": 0.11535, - "38": 0.11465, - "39": 0.1141, - "40": 0.11426, - "41": 0.11528, - "42": 0.11404, - "43": 0.11449, - "44": 0.11494, - "45": 0.11454, - "46": 0.11461, - "47": 0.11428, - "48": 0.11428, - "49": 0.1146, - "50": 0.11407, - "51": 0.42941, - "52": 0.15898, - "53": 0.11765, - "54": 0.11622, - "55": 0.115, - "56": 0.11392, - "57": 0.1136, - "58": 0.11514, - "59": 0.11476, - "60": 0.11475, - "61": 0.11641, - "62": 0.11571, - "63": 0.11492, - "64": 0.1159, - "65": 0.11742, - "66": 0.11593, - "67": 0.11508, - "68": 0.1158, - "69": 0.11605, - "70": 0.11559, - "71": 0.115, - "72": 0.11536, - "73": 0.11556, - "74": 0.11558, - "75": 0.11514, - "76": 0.11555, - "77": 0.11528, - "78": 0.11585, - "79": 0.1158, - "80": 0.11604, - "81": 0.11586, - "82": 0.11595, - "83": 0.11563, - "84": 0.11608, - "85": 0.11609, - "86": 0.11575, - "87": 0.11576, - "88": 0.1155, - "89": 0.11622, - "90": 0.11599, - "91": 0.1157, - "92": 0.116, - "93": 0.11598, - "94": 0.11548, - "95": 0.1162, - "96": 0.1158, - "97": 0.11585, - "98": 0.11589, - "99": 0.11608, - "100": 0.11576 + "2": 3.35114, + "3": 0.13092, + "4": 0.11367, + "5": 0.11255, + "6": 0.11214, + "7": 0.11171, + "8": 0.11387, + "9": 0.11258, + "10": 0.1143, + "11": 0.11391, + "12": 0.11364, + "13": 0.11057, + "14": 0.1115, + "15": 0.11098, + "16": 0.18048, + "17": 0.12969, + "18": 0.11198, + "19": 0.12036, + "20": 0.12086, + "21": 0.12191, + "22": 0.12886, + "23": 0.11986, + "24": 0.12246, + "25": 0.12218, + "26": 0.12061, + "27": 0.12213, + "28": 0.12144, + "29": 0.12199, + "30": 0.12261, + "31": 0.12001, + "32": 0.12346, + "33": 0.12652, + "34": 0.12158, + "35": 0.1204, + "36": 0.12266, + "37": 0.1221, + "38": 0.12261, + "39": 0.12163, + "40": 0.12103, + "41": 0.12169, + "42": 0.12202, + "43": 0.1217, + "44": 0.12812, + "45": 0.12245, + "46": 0.1221, + "47": 0.12253, + "48": 0.12153, + "49": 0.12017, + "50": 0.12139, + "51": 0.20707, + "52": 0.16025, + "53": 0.13024, + "54": 0.12345, + "55": 0.12383, + "56": 0.12376, + "57": 0.12169, + "58": 0.12235, + "59": 0.12205, + "60": 0.12202, + "61": 0.12188, + "62": 0.12073, + "63": 0.1205, + "64": 0.12572, + "65": 0.11988, + "66": 0.12056, + "67": 0.12107, + "68": 0.12137, + "69": 0.12076, + "70": 0.12141, + "71": 0.1194, + "72": 0.1203, + "73": 0.12099, + "74": 0.1204, + "75": 0.12053, + "76": 0.121, + "77": 0.12076, + "78": 0.12727, + "79": 0.11907, + "80": 0.1191, + "81": 0.1209, + "82": 0.1208, + "83": 0.12027, + "84": 0.12137, + "85": 0.11984, + "86": 0.12054, + "87": 0.12076, + "88": 0.12087, + "89": 0.12051, + "90": 0.11984, + "91": 0.12, + "92": 0.12656, + "93": 0.12021, + "94": 0.12136, + "95": 0.12114, + "96": 0.12072, + "97": 0.11932, + "98": 0.12073, + "99": 0.12037, + "100": 0.12128 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2045.0, - "17": 2170.0, + "16": 2096.0, + "17": 2138.0, "18": "nan", - "19": 1954.0, - "20": 2180.0, - "21": 2114.0, - "22": 2061.0, - "23": 2080.0, - "24": 2290.0, - "25": 2107.0, - "26": 2077.0, - "27": 2053.0, - "28": 2127.0, - "29": 2101.0, - "30": 1986.0, - "31": 2234.0, - "32": 2153.0, - "33": 1984.0, - "34": 2217.0, - "35": 2236.0, - "36": 2178.0, - "37": 2181.0, - "38": 2115.0, - "39": 2141.0, - "40": 2393.0, - "41": 2065.0, - "42": 1960.0, - "43": 2164.0, - "44": 2049.0, - "45": 2318.0, - "46": 2163.0, - "47": 2162.0, - "48": 2259.0, - "49": 2370.0, - "50": 2314.0, - "51": 2211.0, - "52": 2104.0, - "53": 2373.0, - "54": 2116.0, - "55": 2199.0, - "56": 2179.0, - "57": 2138.0, - "58": 2684.0, - "59": 2328.0, - "60": 2400.0, - "61": 2143.0, - "62": 2755.0, - "63": 2898.0, - "64": 2685.0, - "65": 2683.0, - "66": 2839.0, - "67": 2825.0, - "68": 2947.0, - "69": 2642.0, - "70": 2858.0, - "71": 2698.0, - "72": 2755.0, - "73": 3032.0, - "74": 3222.0, - "75": 3303.0, - "76": 3371.0, - "77": 3568.0, - "78": 3192.0, - "79": 4016.0, - "80": 3633.0, - "81": 4043.0, - "82": 2660.0, - "83": 3572.0, - "84": 3642.0, - "85": 3648.0, - "86": 3163.0, - "87": 2479.0, - "88": 3242.0, - "89": 3703.0, - "90": 4735.0, - "91": 2605.0, - "92": 3389.0, - "93": 3607.0, - "94": 2731.0, - "95": 3131.0, - "96": 3979.0, - "97": 3416.0, - "98": 3597.0, - "99": 2983.0, - "100": 2860.0 + "19": 2031.0, + "20": 2173.0, + "21": 2175.0, + "22": 2056.0, + "23": 2037.0, + "24": 2353.0, + "25": 2037.0, + "26": 2181.0, + "27": 2211.0, + "28": 2128.0, + "29": 2128.0, + "30": 1960.0, + "31": 2238.0, + "32": 2207.0, + "33": 1999.0, + "34": 2274.0, + "35": 2218.0, + "36": 2051.0, + "37": 2246.0, + "38": 2037.0, + "39": 2200.0, + "40": 2354.0, + "41": 2045.0, + "42": 1947.0, + "43": 2288.0, + "44": 2035.0, + "45": 2441.0, + "46": 2199.0, + "47": 2204.0, + "48": 2321.0, + "49": 2357.0, + "50": 2362.0, + "51": 2183.0, + "52": 2236.0, + "53": 2368.0, + "54": 2073.0, + "55": 2338.0, + "56": 2214.0, + "57": 2167.0, + "58": 2705.0, + "59": 2408.0, + "60": 2384.0, + "61": 2222.0, + "62": 2714.0, + "63": 2838.0, + "64": 2703.0, + "65": 2660.0, + "66": 2777.0, + "67": 2861.0, + "68": 2828.0, + "69": 2628.0, + "70": 2889.0, + "71": 2731.0, + "72": 2714.0, + "73": 2971.0, + "74": 3230.0, + "75": 3265.0, + "76": 3337.0, + "77": 3584.0, + "78": 3220.0, + "79": 3852.0, + "80": 3490.0, + "81": 3976.0, + "82": 2598.0, + "83": 3671.0, + "84": 3564.0, + "85": 3588.0, + "86": 3191.0, + "87": 2511.0, + "88": 3226.0, + "89": 3851.0, + "90": 4692.0, + "91": 2593.0, + "92": 3421.0, + "93": 3588.0, + "94": 2678.0, + "95": 3098.0, + "96": 4088.0, + "97": 3444.0, + "98": 3620.0, + "99": 2998.0, + "100": 2906.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json index 465958f242a..d485c8dfd3a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -4,105 +4,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90482, - "2": 10.91127, - "3": 10.91389, - "4": 10.91125, - "5": 10.90415, - "6": 10.88705, + "1": 10.90478, + "2": 10.91126, + "3": 10.91387, + "4": 10.91128, + "5": 10.90416, + "6": 10.88704, "7": 10.89958, - "8": 10.90974, - "9": 10.90876, - "10": 10.90909, - "11": 10.89892, - "12": 10.90381, - "13": 10.89481, - "14": 10.895, - "15": 10.91151, - "16": 10.90212, - "17": 10.90521, - "18": 10.90266, - "19": 10.89858, - "20": 10.88144, - "21": 10.90599, - "22": 10.90541, - "23": 10.89775, - "24": 10.87035, - "25": 10.88754, + "8": 10.9097, + "9": 10.90881, + "10": 10.90907, + "11": 10.89893, + "12": 10.90379, + "13": 10.8948, + "14": 10.89499, + "15": 10.91148, + "16": 10.90213, + "17": 10.90519, + "18": 10.90263, + "19": 10.89853, + "20": 10.88141, + "21": 10.906, + "22": 10.9054, + "23": 10.89771, + "24": 10.8703, + "25": 10.88755, "26": 10.87502, - "27": 10.85541, + "27": 10.85537, "28": 10.84417, - "29": 10.83902, - "30": 10.82444, - "31": 10.81264, - "32": 10.79356, - "33": 10.78745, - "34": 10.74178, - "35": 10.73997, - "36": 10.72531, + "29": 10.83904, + "30": 10.82443, + "31": 10.81259, + "32": 10.79357, + "33": 10.78738, + "34": 10.74174, + "35": 10.73995, + "36": 10.7253, "37": 10.69288, - "38": 10.6855, - "39": 10.65581, - "40": 10.64123, - "41": 10.61511, - "42": 10.59759, + "38": 10.68548, + "39": 10.6558, + "40": 10.6412, + "41": 10.61508, + "42": 10.59757, "43": 10.57142, - "44": 10.54295, - "45": 10.54104, - "46": 10.5127, - "47": 10.49103, - "48": 10.44312, - "49": 10.43074, - "50": 10.42583, - "51": 10.40703, - "52": 10.36347, - "53": 10.36714, - "54": 10.32652, - "55": 10.285, + "44": 10.54297, + "45": 10.54102, + "46": 10.51267, + "47": 10.491, + "48": 10.4431, + "49": 10.43073, + "50": 10.42579, + "51": 10.40702, + "52": 10.36344, + "53": 10.36712, + "54": 10.3265, + "55": 10.28501, "56": 10.30875, - "57": 10.28491, - "58": 10.28691, - "59": 10.23535, + "57": 10.2849, + "58": 10.28689, + "59": 10.23531, "60": 10.23954, "61": 10.18706, "62": 10.15381, "63": 10.20991, "64": 10.16587, - "65": 10.14344, - "66": 10.15538, + "65": 10.14343, + "66": 10.15536, "67": 10.13045, "68": 10.0927, "69": 10.11275, - "70": 10.0928, - "71": 10.10124, - "72": 10.09019, - "73": 10.07835, - "74": 10.06408, - "75": 10.03188, - "76": 10.06024, - "77": 10.05889, - "78": 10.01235, - "79": 10.02221, - "80": 10.04027, - "81": 10.05412, - "82": 9.99136, + "70": 10.09278, + "71": 10.10122, + "72": 10.09018, + "73": 10.07833, + "74": 10.06406, + "75": 10.03187, + "76": 10.06022, + "77": 10.05888, + "78": 10.01234, + "79": 10.02218, + "80": 10.04025, + "81": 10.05411, + "82": 9.99134, "83": 9.97163, - "84": 9.9039, - "85": 9.89913, - "86": 9.99355, - "87": 10.0053, - "88": 9.98909, - "89": 9.91902, - "90": 9.91567, + "84": 9.90388, + "85": 9.89912, + "86": 9.99353, + "87": 10.00529, + "88": 9.98908, + "89": 9.919, + "90": 9.91566, "91": 9.94645, - "92": 9.92276, - "93": 9.85617, + "92": 9.92275, + "93": 9.85615, "94": 9.93303, - "95": 9.92323, - "96": 9.89867, - "97": 9.84837, + "95": 9.92322, + "96": 9.89864, + "97": 9.84836, "98": 9.87267, - "99": 9.91403, + "99": 9.91402, "100": 9.81536 } }, @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.69795, - "3": 0.12822, - "4": 0.11191, - "5": 0.11305, - "6": 0.11291, - "7": 0.11148, - "8": 0.11017, - "9": 0.11088, - "10": 0.11178, - "11": 0.11142, - "12": 0.11218, - "13": 0.11237, - "14": 0.1129, - "15": 0.11262, - "16": 0.11247, - "17": 0.1927, - "18": 0.13346, - "19": 0.12517, - "20": 0.12457, - "21": 0.12371, - "22": 0.12308, - "23": 0.12377, - "24": 0.12334, - "25": 0.12305, - "26": 0.12418, - "27": 0.12292, - "28": 0.12436, - "29": 0.12391, - "30": 0.12441, - "31": 0.12325, - "32": 0.12553, - "33": 0.1258, - "34": 0.12453, - "35": 0.12383, - "36": 0.12352, - "37": 0.12407, - "38": 0.12364, - "39": 0.12387, - "40": 0.12359, - "41": 0.12366, - "42": 0.12425, - "43": 0.12415, - "44": 0.12445, - "45": 0.12496, - "46": 0.12427, - "47": 0.12418, - "48": 0.12305, - "49": 0.12383, - "50": 0.12316, - "51": 0.25848, - "52": 0.13565, - "53": 0.15181, - "54": 0.12467, - "55": 0.12484, - "56": 0.12402, - "57": 0.12403, - "58": 0.12421, - "59": 0.12433, - "60": 0.12445, - "61": 0.12425, - "62": 0.12333, - "63": 0.1237, - "64": 0.12367, - "65": 0.12388, - "66": 0.12372, - "67": 0.12373, - "68": 0.12385, - "69": 0.12363, - "70": 0.1231, - "71": 0.12411, - "72": 0.12389, - "73": 0.12327, - "74": 0.12399, - "75": 0.12344, - "76": 0.12364, - "77": 0.12362, - "78": 0.1244, - "79": 0.12429, - "80": 0.12426, - "81": 0.12456, - "82": 0.12384, - "83": 0.12405, - "84": 0.12419, - "85": 0.12465, - "86": 0.1243, - "87": 0.12412, - "88": 0.12441, - "89": 0.12403, - "90": 0.1239, - "91": 0.12399, - "92": 0.12416, - "93": 0.1242, - "94": 0.12451, - "95": 0.1223, - "96": 0.1228, - "97": 0.12256, - "98": 0.12281, - "99": 0.1239, - "100": 0.12528 + "2": 6.63834, + "3": 0.13377, + "4": 0.09442, + "5": 0.09288, + "6": 0.09236, + "7": 0.09186, + "8": 0.09218, + "9": 0.09293, + "10": 0.09251, + "11": 0.09249, + "12": 0.09185, + "13": 0.0911, + "14": 0.09273, + "15": 0.09212, + "16": 0.09196, + "17": 0.16777, + "18": 0.11036, + "19": 0.10128, + "20": 0.10062, + "21": 0.11518, + "22": 0.09898, + "23": 0.09942, + "24": 0.09954, + "25": 0.1013, + "26": 0.09883, + "27": 0.09876, + "28": 0.0988, + "29": 0.09925, + "30": 0.09887, + "31": 0.0992, + "32": 0.1004, + "33": 0.09873, + "34": 0.10228, + "35": 0.1025, + "36": 0.10115, + "37": 0.10121, + "38": 0.10043, + "39": 0.09928, + "40": 0.09968, + "41": 0.10021, + "42": 0.0991, + "43": 0.09889, + "44": 0.09921, + "45": 0.09874, + "46": 0.09914, + "47": 0.10043, + "48": 0.09964, + "49": 0.09802, + "50": 0.09774, + "51": 0.27565, + "52": 0.14227, + "53": 0.10227, + "54": 0.10029, + "55": 0.1004, + "56": 0.1, + "57": 0.09848, + "58": 0.0999, + "59": 0.09933, + "60": 0.09885, + "61": 0.09913, + "62": 0.10018, + "63": 0.09976, + "64": 0.09967, + "65": 0.10083, + "66": 0.09912, + "67": 0.10081, + "68": 0.09992, + "69": 0.0994, + "70": 0.09892, + "71": 0.09939, + "72": 0.10029, + "73": 0.09974, + "74": 0.09884, + "75": 0.10073, + "76": 0.10056, + "77": 0.10115, + "78": 0.0999, + "79": 0.09911, + "80": 0.10014, + "81": 0.09916, + "82": 0.09932, + "83": 0.10092, + "84": 0.09949, + "85": 0.09955, + "86": 0.09966, + "87": 0.10196, + "88": 0.09937, + "89": 0.09933, + "90": 0.10017, + "91": 0.12244, + "92": 0.10035, + "93": 0.09918, + "94": 0.10188, + "95": 0.1009, + "96": 0.09982, + "97": 0.0997, + "98": 0.12767, + "99": 0.10287, + "100": 0.1035 } }, "num-zeros": { @@ -448,90 +448,90 @@ "14": "nan", "15": "nan", "16": "nan", - "17": 2245.0, - "18": 2424.0, - "19": 1985.0, - "20": 2123.0, - "21": 2167.0, - "22": 2033.0, - "23": 1981.0, - "24": 2224.0, - "25": 2110.0, - "26": 2146.0, - "27": 2088.0, - "28": 2003.0, - "29": 2073.0, - "30": 1899.0, - "31": 2214.0, - "32": 2083.0, - "33": 1911.0, - "34": 2241.0, - "35": 2346.0, - "36": 1942.0, - "37": 2276.0, - "38": 2254.0, - "39": 2106.0, - "40": 2357.0, - "41": 2031.0, - "42": 1980.0, - "43": 2163.0, - "44": 1964.0, - "45": 2451.0, - "46": 2266.0, - "47": 2202.0, - "48": 2354.0, - "49": 2437.0, - "50": 2220.0, - "51": 2297.0, - "52": 2279.0, - "53": 2322.0, - "54": 2194.0, - "55": 2271.0, - "56": 2317.0, - "57": 2177.0, - "58": 2787.0, - "59": 2486.0, - "60": 2516.0, - "61": 2315.0, - "62": 2876.0, - "63": 2984.0, - "64": 2788.0, - "65": 2648.0, - "66": 2829.0, - "67": 2885.0, - "68": 3025.0, - "69": 2740.0, - "70": 2838.0, - "71": 2720.0, - "72": 2827.0, - "73": 3061.0, - "74": 3277.0, - "75": 3355.0, - "76": 3240.0, - "77": 3527.0, - "78": 3134.0, - "79": 3983.0, - "80": 3418.0, - "81": 3873.0, - "82": 2672.0, - "83": 3457.0, - "84": 3338.0, - "85": 3515.0, - "86": 3182.0, - "87": 2513.0, - "88": 3475.0, - "89": 3858.0, - "90": 4613.0, - "91": 2473.0, - "92": 3393.0, - "93": 3508.0, - "94": 2892.0, - "95": 2943.0, - "96": 3986.0, - "97": 3909.0, - "98": 3893.0, - "99": 2879.0, - "100": 3136.0 + "17": 2230.0, + "18": 2296.0, + "19": 1911.0, + "20": 2144.0, + "21": 2182.0, + "22": 1981.0, + "23": 1998.0, + "24": 2249.0, + "25": 2133.0, + "26": 2163.0, + "27": 2108.0, + "28": 2068.0, + "29": 2131.0, + "30": 1874.0, + "31": 2149.0, + "32": 2121.0, + "33": 2014.0, + "34": 2237.0, + "35": 2283.0, + "36": 2025.0, + "37": 2195.0, + "38": 2100.0, + "39": 2123.0, + "40": 2287.0, + "41": 1932.0, + "42": 1968.0, + "43": 2231.0, + "44": 2052.0, + "45": 2443.0, + "46": 2224.0, + "47": 2154.0, + "48": 2328.0, + "49": 2408.0, + "50": 2325.0, + "51": 2288.0, + "52": 2320.0, + "53": 2415.0, + "54": 2229.0, + "55": 2269.0, + "56": 2318.0, + "57": 2246.0, + "58": 2768.0, + "59": 2331.0, + "60": 2465.0, + "61": 2314.0, + "62": 2794.0, + "63": 2991.0, + "64": 2854.0, + "65": 2589.0, + "66": 2801.0, + "67": 2961.0, + "68": 2903.0, + "69": 2858.0, + "70": 2884.0, + "71": 2736.0, + "72": 2801.0, + "73": 3017.0, + "74": 3394.0, + "75": 3365.0, + "76": 3272.0, + "77": 3530.0, + "78": 3130.0, + "79": 3903.0, + "80": 3534.0, + "81": 3942.0, + "82": 2695.0, + "83": 3502.0, + "84": 3278.0, + "85": 3481.0, + "86": 3082.0, + "87": 2608.0, + "88": 3334.0, + "89": 3927.0, + "90": 4674.0, + "91": 2507.0, + "92": 3332.0, + "93": 3568.0, + "94": 2850.0, + "95": 2987.0, + "96": 3863.0, + "97": 3854.0, + "98": 3962.0, + "99": 2963.0, + "100": 3127.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json index a5014b09b8c..e6a1ba4c777 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92747, - "2": 10.92329, - "3": 10.93106, - "4": 10.93351, - "5": 10.92989, - "6": 10.92212, - "7": 10.93386, - "8": 10.92351, - "9": 10.92811, - "10": 10.92867, - "11": 10.9169, - "12": 10.92391, - "13": 10.91775, - "14": 10.9069, - "15": 10.88243, - "16": 10.87691, - "17": 10.88734, - "18": 10.86241, + "1": 10.92737, + "2": 10.92322, + "3": 10.93102, + "4": 10.93361, + "5": 10.92983, + "6": 10.92207, + "7": 10.93393, + "8": 10.92358, + "9": 10.92812, + "10": 10.9287, + "11": 10.91696, + "12": 10.92387, + "13": 10.91767, + "14": 10.90691, + "15": 10.88252, + "16": 10.8769, + "17": 10.88742, + "18": 10.86238, "19": 10.87075, - "20": 10.78063, - "21": 10.78168, - "22": 10.76713, - "23": 10.76218, - "24": 10.72713, - "25": 10.73375, - "26": 10.71069, - "27": 10.67877, - "28": 10.5997, - "29": 10.5811, + "20": 10.78076, + "21": 10.78172, + "22": 10.76719, + "23": 10.7623, + "24": 10.72715, + "25": 10.73371, + "26": 10.71078, + "27": 10.67879, + "28": 10.59968, + "29": 10.58113, "30": 10.5552, - "31": 10.55952, - "32": 10.53207, - "33": 10.5001, - "34": 10.45962, - "35": 10.47444, + "31": 10.55962, + "32": 10.53209, + "33": 10.50011, + "34": 10.45968, + "35": 10.47452, "36": 10.45368, - "37": 10.41546, - "38": 10.41354, - "39": 10.37796, - "40": 10.37129, - "41": 10.33976, - "42": 10.32738, - "43": 10.30585, - "44": 10.27175, - "45": 10.29766, - "46": 10.25645, - "47": 10.2346, - "48": 10.19031, - "49": 10.1819, - "50": 10.19451 + "37": 10.41553, + "38": 10.41358, + "39": 10.37802, + "40": 10.37132, + "41": 10.33979, + "42": 10.32744, + "43": 10.30593, + "44": 10.2718, + "45": 10.29772, + "46": 10.25649, + "47": 10.23463, + "48": 10.19034, + "49": 10.18191, + "50": 10.19455 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1758.0, - "2": 1704.0, - "3": 1903.0, - "4": 1731.0, - "5": 1771.0, - "6": 1713.0, - "7": 1891.0, - "8": 1706.0, - "9": 1876.0, - "10": 1715.0, - "11": 1780.0, - "12": 1697.0, - "13": 1781.0, - "14": 1990.0, - "15": 1614.0, - "16": 1720.0, - "17": 1908.0, - "18": 1862.0, - "19": 1724.0, - "20": 1739.0, - "21": 1784.0, - "22": 1735.0, - "23": 1666.0, - "24": 1788.0, - "25": 1706.0, - "26": 1812.0, - "27": 1847.0, - "28": 1837.0, - "29": 1956.0, - "30": 1835.0, - "31": 2006.0, - "32": 1917.0, - "33": 1907.0, - "34": 2089.0, - "35": 2077.0, - "36": 2012.0, - "37": 2253.0, - "38": 2177.0, - "39": 2254.0, - "40": 2227.0, - "41": 2409.0, - "42": 1981.0, - "43": 2388.0, - "44": 2186.0, - "45": 2570.0, - "46": 2403.0, - "47": 2415.0, - "48": 2543.0, - "49": 2918.0, - "50": 2627.0 + "1": 1715.0, + "2": 1761.0, + "3": 1831.0, + "4": 1692.0, + "5": 1745.0, + "6": 1733.0, + "7": 1882.0, + "8": 1735.0, + "9": 1860.0, + "10": 1783.0, + "11": 1721.0, + "12": 1764.0, + "13": 1834.0, + "14": 2000.0, + "15": 1662.0, + "16": 1731.0, + "17": 1825.0, + "18": 1766.0, + "19": 1728.0, + "20": 1717.0, + "21": 1740.0, + "22": 1761.0, + "23": 1692.0, + "24": 1843.0, + "25": 1682.0, + "26": 1866.0, + "27": 1801.0, + "28": 1801.0, + "29": 1909.0, + "30": 1847.0, + "31": 2000.0, + "32": 1986.0, + "33": 1850.0, + "34": 2040.0, + "35": 2003.0, + "36": 1995.0, + "37": 2170.0, + "38": 2063.0, + "39": 2295.0, + "40": 2268.0, + "41": 2402.0, + "42": 2017.0, + "43": 2365.0, + "44": 2172.0, + "45": 2498.0, + "46": 2299.0, + "47": 2490.0, + "48": 2644.0, + "49": 2822.0, + "50": 2464.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.62642, - "3": 0.16438, - "4": 0.15317, - "5": 0.15067, - "6": 0.15109, - "7": 0.15071, - "8": 0.15197, - "9": 0.15102, - "10": 0.15115, - "11": 0.14979, - "12": 0.14961, - "13": 0.15008, - "14": 0.15, - "15": 0.15032, - "16": 0.15005, - "17": 0.15091, - "18": 0.15158, - "19": 0.15086, - "20": 0.15166, - "21": 0.1499, - "22": 0.15039, - "23": 0.15101, - "24": 0.15006, - "25": 0.15063, - "26": 0.14879, - "27": 0.15022, - "28": 0.15025, - "29": 0.15108, - "30": 0.15091, - "31": 0.15094, - "32": 0.15086, - "33": 0.1511, - "34": 0.15116, - "35": 0.15062, - "36": 0.15132, - "37": 0.15085, - "38": 0.15063, - "39": 0.15107, - "40": 0.15224, - "41": 0.15186, - "42": 0.15218, - "43": 0.15305, - "44": 0.15195, - "45": 0.15194, - "46": 0.1522, - "47": 0.15172, - "48": 0.15314, - "49": 0.15138, - "50": 0.15163 + "2": 8.54195, + "3": 0.16611, + "4": 0.12546, + "5": 0.12368, + "6": 0.12226, + "7": 0.12236, + "8": 0.12326, + "9": 0.12297, + "10": 0.1246, + "11": 0.12334, + "12": 0.12432, + "13": 0.12342, + "14": 0.12224, + "15": 0.12272, + "16": 0.12405, + "17": 0.12416, + "18": 0.12374, + "19": 0.12346, + "20": 0.1233, + "21": 0.12363, + "22": 0.12471, + "23": 0.12449, + "24": 0.12425, + "25": 0.12518, + "26": 0.12432, + "27": 0.12341, + "28": 0.12368, + "29": 0.12436, + "30": 0.12304, + "31": 0.12364, + "32": 0.12362, + "33": 0.12585, + "34": 0.1263, + "35": 0.22497, + "36": 0.12387, + "37": 0.12458, + "38": 0.12354, + "39": 0.12559, + "40": 0.12704, + "41": 0.12769, + "42": 0.12543, + "43": 0.12513, + "44": 0.12408, + "45": 0.125, + "46": 0.1257, + "47": 0.12435, + "48": 0.12526, + "49": 0.12532, + "50": 0.12483 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_h100.json index 3e89a6dc506..3b924239e5d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.9595, - "2": 10.94535, - "3": 10.94822, - "4": 10.94596, - "5": 10.95573, - "6": 10.95055, - "7": 10.94891, - "8": 10.95521, - "9": 10.94248, - "10": 10.94574, - "11": 10.9347, - "12": 10.94598, + "1": 10.95958, + "2": 10.94531, + "3": 10.94826, + "4": 10.94593, + "5": 10.95572, + "6": 10.95044, + "7": 10.94896, + "8": 10.95527, + "9": 10.9425, + "10": 10.94576, + "11": 10.93469, + "12": 10.946, "13": 10.91934, - "14": 10.92006, - "15": 10.90317, - "16": 10.89124, + "14": 10.92002, + "15": 10.90323, + "16": 10.8913, "17": 10.8913, - "18": 10.89155, - "19": 10.87822, - "20": 10.80519, - "21": 10.78699, - "22": 10.77636, - "23": 10.7802, - "24": 10.7577, + "18": 10.8916, + "19": 10.87816, + "20": 10.8052, + "21": 10.78706, + "22": 10.77645, + "23": 10.7803, + "24": 10.75769, "25": 10.75525, - "26": 10.73171, - "27": 10.69473, - "28": 10.60826, - "29": 10.57758, - "30": 10.56278, - "31": 10.56266, - "32": 10.5491, - "33": 10.51923, - "34": 10.4687, - "35": 10.48064, - "36": 10.45916, - "37": 10.4221, - "38": 10.41763, + "26": 10.73175, + "27": 10.69476, + "28": 10.60836, + "29": 10.57765, + "30": 10.56276, + "31": 10.56271, + "32": 10.54921, + "33": 10.51927, + "34": 10.46879, + "35": 10.48072, + "36": 10.45917, + "37": 10.42218, + "38": 10.41772, "39": 10.38849, - "40": 10.36988, - "41": 10.34987, - "42": 10.34711, - "43": 10.31394, - "44": 10.30201, - "45": 10.29384, - "46": 10.26322, - "47": 10.25235, - "48": 10.20727, + "40": 10.36985, + "41": 10.3499, + "42": 10.34719, + "43": 10.31399, + "44": 10.30199, + "45": 10.29388, + "46": 10.2633, + "47": 10.25232, + "48": 10.2073, "49": 10.19925, - "50": 10.21455 + "50": 10.2145 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1733.0, - "2": 1714.0, - "3": 1829.0, - "4": 1672.0, - "5": 1686.0, - "6": 1763.0, - "7": 1863.0, - "8": 1736.0, - "9": 1739.0, - "10": 1754.0, - "11": 1661.0, - "12": 1691.0, - "13": 1822.0, - "14": 1831.0, - "15": 1647.0, - "16": 1677.0, - "17": 1778.0, - "18": 1743.0, - "19": 1712.0, - "20": 1615.0, - "21": 1824.0, - "22": 1685.0, - "23": 1680.0, - "24": 1751.0, - "25": 1662.0, - "26": 1697.0, - "27": 1874.0, - "28": 1841.0, - "29": 1847.0, - "30": 1855.0, - "31": 2009.0, - "32": 1997.0, - "33": 2040.0, - "34": 2050.0, - "35": 2014.0, - "36": 1986.0, - "37": 2199.0, - "38": 1990.0, - "39": 2259.0, - "40": 2336.0, - "41": 2377.0, - "42": 2067.0, - "43": 2353.0, - "44": 2189.0, - "45": 2467.0, - "46": 2453.0, - "47": 2516.0, - "48": 2655.0, - "49": 2795.0, - "50": 2314.0 + "1": 1758.0, + "2": 1761.0, + "3": 1813.0, + "4": 1675.0, + "5": 1720.0, + "6": 1626.0, + "7": 1904.0, + "8": 1666.0, + "9": 1712.0, + "10": 1725.0, + "11": 1604.0, + "12": 1606.0, + "13": 1740.0, + "14": 1779.0, + "15": 1655.0, + "16": 1754.0, + "17": 1842.0, + "18": 1719.0, + "19": 1738.0, + "20": 1650.0, + "21": 1783.0, + "22": 1686.0, + "23": 1605.0, + "24": 1822.0, + "25": 1673.0, + "26": 1762.0, + "27": 1825.0, + "28": 1742.0, + "29": 1879.0, + "30": 1744.0, + "31": 1987.0, + "32": 1943.0, + "33": 1943.0, + "34": 2011.0, + "35": 1978.0, + "36": 2019.0, + "37": 2124.0, + "38": 2058.0, + "39": 2219.0, + "40": 2302.0, + "41": 2426.0, + "42": 2000.0, + "43": 2352.0, + "44": 2251.0, + "45": 2494.0, + "46": 2447.0, + "47": 2530.0, + "48": 2566.0, + "49": 2819.0, + "50": 2468.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 777900032.0, - "2": 777900032.0, - "3": 777900032.0, - "4": 777900032.0, - "5": 777900032.0, - "6": 777900032.0, - "7": 777900032.0, - "8": 777900032.0, - "9": 777900032.0, - "10": 777900032.0, - "11": 777900032.0, - "12": 777900032.0, - "13": 777900032.0, - "14": 777900032.0, - "15": 777900032.0, - "16": 777900032.0, - "17": 777900032.0, - "18": 777900032.0, - "19": 777900032.0, - "20": 777900032.0, - "21": 777900032.0, - "22": 777900032.0, - "23": 777900032.0, - "24": 777900032.0, - "25": 777900032.0, - "26": 777900032.0, - "27": 777900032.0, - "28": 777900032.0, - "29": 777900032.0, - "30": 777900032.0, - "31": 777900032.0, - "32": 777900032.0, - "33": 777900032.0, - "34": 777900032.0, - "35": 777900032.0, - "36": 777900032.0, - "37": 777900032.0, - "38": 777900032.0, - "39": 777900032.0, - "40": 777900032.0, - "41": 777900032.0, - "42": 777900032.0, - "43": 777900032.0, - "44": 777900032.0, - "45": 777900032.0, - "46": 777900032.0, - "47": 777900032.0, - "48": 777900032.0, - "49": 777900032.0, - "50": 777900032.0 + "1": 778948608.0, + "2": 778948608.0, + "3": 778948608.0, + "4": 778948608.0, + "5": 778948608.0, + "6": 778948608.0, + "7": 778948608.0, + "8": 778948608.0, + "9": 778948608.0, + "10": 778948608.0, + "11": 778948608.0, + "12": 778948608.0, + "13": 778948608.0, + "14": 778948608.0, + "15": 778948608.0, + "16": 778948608.0, + "17": 778948608.0, + "18": 778948608.0, + "19": 778948608.0, + "20": 778948608.0, + "21": 778948608.0, + "22": 778948608.0, + "23": 778948608.0, + "24": 778948608.0, + "25": 778948608.0, + "26": 778948608.0, + "27": 778948608.0, + "28": 778948608.0, + "29": 778948608.0, + "30": 778948608.0, + "31": 778948608.0, + "32": 778948608.0, + "33": 778948608.0, + "34": 778948608.0, + "35": 778948608.0, + "36": 778948608.0, + "37": 778948608.0, + "38": 778948608.0, + "39": 778948608.0, + "40": 778948608.0, + "41": 778948608.0, + "42": 778948608.0, + "43": 778948608.0, + "44": 778948608.0, + "45": 778948608.0, + "46": 778948608.0, + "47": 778948608.0, + "48": 778948608.0, + "49": 778948608.0, + "50": 778948608.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2463815680.0, - "2": 2744478720.0, - "3": 2744478720.0, - "4": 2744478720.0, - "5": 2744478720.0, - "6": 2744478720.0, - "7": 2744478720.0, - "8": 2744478720.0, - "9": 2744478720.0, - "10": 2744478720.0, - "11": 2744478720.0, - "12": 2744478720.0, - "13": 2744478720.0, - "14": 2744478720.0, - "15": 2744478720.0, - "16": 2744478720.0, - "17": 2744478720.0, - "18": 2744478720.0, - "19": 2744478720.0, - "20": 2744478720.0, - "21": 2744478720.0, - "22": 2744478720.0, - "23": 2744478720.0, - "24": 2744478720.0, - "25": 2744478720.0, - "26": 2744478720.0, - "27": 2744478720.0, - "28": 2744478720.0, - "29": 2744478720.0, - "30": 2744478720.0, - "31": 2744478720.0, - "32": 2744478720.0, - "33": 2744478720.0, - "34": 2744478720.0, - "35": 2744478720.0, - "36": 2744478720.0, - "37": 2744478720.0, - "38": 2744478720.0, - "39": 2744478720.0, - "40": 2744478720.0, - "41": 2744478720.0, - "42": 2744478720.0, - "43": 2744478720.0, - "44": 2744478720.0, - "45": 2744478720.0, - "46": 2744478720.0, - "47": 2744478720.0, - "48": 2744478720.0, - "49": 2744478720.0, - "50": 2744478720.0 + "1": 2462767104.0, + "2": 2746575872.0, + "3": 2746575872.0, + "4": 2746575872.0, + "5": 2746575872.0, + "6": 2746575872.0, + "7": 2746575872.0, + "8": 2746575872.0, + "9": 2746575872.0, + "10": 2746575872.0, + "11": 2746575872.0, + "12": 2746575872.0, + "13": 2746575872.0, + "14": 2746575872.0, + "15": 2746575872.0, + "16": 2746575872.0, + "17": 2746575872.0, + "18": 2746575872.0, + "19": 2746575872.0, + "20": 2746575872.0, + "21": 2746575872.0, + "22": 2746575872.0, + "23": 2746575872.0, + "24": 2746575872.0, + "25": 2746575872.0, + "26": 2746575872.0, + "27": 2746575872.0, + "28": 2746575872.0, + "29": 2746575872.0, + "30": 2746575872.0, + "31": 2746575872.0, + "32": 2746575872.0, + "33": 2746575872.0, + "34": 2746575872.0, + "35": 2746575872.0, + "36": 2746575872.0, + "37": 2746575872.0, + "38": 2746575872.0, + "39": 2746575872.0, + "40": 2746575872.0, + "41": 2746575872.0, + "42": 2746575872.0, + "43": 2746575872.0, + "44": 2746575872.0, + "45": 2746575872.0, + "46": 2746575872.0, + "47": 2746575872.0, + "48": 2746575872.0, + "49": 2746575872.0, + "50": 2746575872.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.82976, - "3": 0.11937, - "4": 0.09967, - "5": 0.09729, - "6": 0.09699, - "7": 0.09735, - "8": 0.09734, - "9": 0.09941, - "10": 0.09806, - "11": 0.09745, - "12": 0.09843, - "13": 0.09805, - "14": 0.10028, - "15": 0.09782, - "16": 0.09832, - "17": 0.09864, - "18": 0.09887, - "19": 0.09814, - "20": 0.09872, - "21": 0.0983, - "22": 0.09812, - "23": 0.0986, - "24": 0.09864, - "25": 0.09814, - "26": 0.09842, - "27": 0.09888, - "28": 0.09827, - "29": 0.09853, - "30": 0.09888, - "31": 0.09821, - "32": 0.09887, - "33": 0.09832, - "34": 0.09893, - "35": 0.09849, - "36": 0.09902, - "37": 0.09918, - "38": 0.09829, - "39": 0.09821, - "40": 0.09878, - "41": 0.09893, - "42": 0.09887, - "43": 0.09864, - "44": 0.09855, - "45": 0.1, - "46": 0.09978, - "47": 0.09892, - "48": 0.09876, - "49": 0.09904, - "50": 0.09827 + "2": 6.20121, + "3": 0.12675, + "4": 0.10217, + "5": 0.10281, + "6": 0.10192, + "7": 0.10169, + "8": 0.10179, + "9": 0.10254, + "10": 0.10206, + "11": 0.1024, + "12": 0.10146, + "13": 0.10211, + "14": 0.10213, + "15": 0.10162, + "16": 0.10155, + "17": 0.10153, + "18": 0.10231, + "19": 0.10202, + "20": 0.10293, + "21": 0.10313, + "22": 0.10248, + "23": 0.10223, + "24": 0.10324, + "25": 0.10133, + "26": 0.10185, + "27": 0.10234, + "28": 0.10191, + "29": 0.10192, + "30": 0.10168, + "31": 0.10149, + "32": 0.102, + "33": 0.10228, + "34": 0.10276, + "35": 0.10214, + "36": 0.10173, + "37": 0.10195, + "38": 0.10184, + "39": 0.10317, + "40": 0.10254, + "41": 0.1023, + "42": 0.1017, + "43": 0.10198, + "44": 0.10214, + "45": 0.10227, + "46": 0.10186, + "47": 0.10281, + "48": 0.10252, + "49": 0.10255, + "50": 0.10251 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_a100.json index afc7947f7f0..48a323fa776 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_a100.json @@ -4,105 +4,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92488, - "2": 10.92907, - "3": 10.93082, - "4": 10.93427, - "5": 10.92867, - "6": 10.92578, - "7": 10.94045, - "8": 10.93578, - "9": 10.94395, - "10": 10.93028, - "11": 10.93265, + "1": 10.92487, + "2": 10.92908, + "3": 10.93084, + "4": 10.93423, + "5": 10.9287, + "6": 10.92576, + "7": 10.94046, + "8": 10.93584, + "9": 10.94396, + "10": 10.93026, + "11": 10.93262, "12": 10.93126, - "13": 10.92463, - "14": 10.92175, + "13": 10.92467, + "14": 10.92176, "15": 10.93939, "16": 10.92892, - "17": 10.93643, + "17": 10.93636, "18": 10.93293, - "19": 10.9348, - "20": 10.90488, - "21": 10.93658, - "22": 10.92758, - "23": 10.92938, - "24": 10.89949, - "25": 10.90999, - "26": 10.87681, - "27": 10.87086, - "28": 10.84663, - "29": 10.85001, - "30": 10.83447, - "31": 10.8246, - "32": 10.80967, - "33": 10.80674, - "34": 10.75948, + "19": 10.93483, + "20": 10.90491, + "21": 10.93654, + "22": 10.92755, + "23": 10.9294, + "24": 10.89951, + "25": 10.90996, + "26": 10.87686, + "27": 10.87087, + "28": 10.8466, + "29": 10.85002, + "30": 10.8345, + "31": 10.82458, + "32": 10.80964, + "33": 10.80677, + "34": 10.75947, "35": 10.75967, - "36": 10.74798, - "37": 10.71577, + "36": 10.74793, + "37": 10.71574, "38": 10.69993, - "39": 10.67296, - "40": 10.65179, - "41": 10.63083, - "42": 10.60495, - "43": 10.5774, - "44": 10.5624, - "45": 10.5548, + "39": 10.67298, + "40": 10.65175, + "41": 10.63082, + "42": 10.60496, + "43": 10.57739, + "44": 10.56235, + "45": 10.55478, "46": 10.5173, - "47": 10.49532, - "48": 10.45853, - "49": 10.43585, + "47": 10.49531, + "48": 10.4585, + "49": 10.43581, "50": 10.43575, "51": 10.41931, - "52": 10.37454, - "53": 10.36254, - "54": 10.34598, - "55": 10.29852, - "56": 10.31156, + "52": 10.37453, + "53": 10.36252, + "54": 10.34596, + "55": 10.2985, + "56": 10.31153, "57": 10.29561, - "58": 10.2978, - "59": 10.23625, + "58": 10.29779, + "59": 10.23624, "60": 10.25226, - "61": 10.19881, - "62": 10.16881, - "63": 10.21472, - "64": 10.16225, - "65": 10.14193, - "66": 10.15758, - "67": 10.13537, - "68": 10.09927, - "69": 10.12128, - "70": 10.08613, - "71": 10.10776, - "72": 10.08726, - "73": 10.07923, - "74": 10.06518, - "75": 10.03132, - "76": 10.05011, - "77": 10.05532, - "78": 10.01826, - "79": 10.01821, - "80": 10.0367, + "61": 10.1988, + "62": 10.16878, + "63": 10.21471, + "64": 10.16226, + "65": 10.14191, + "66": 10.15755, + "67": 10.13536, + "68": 10.09925, + "69": 10.12127, + "70": 10.08611, + "71": 10.10774, + "72": 10.08727, + "73": 10.0792, + "74": 10.06515, + "75": 10.03131, + "76": 10.05009, + "77": 10.05533, + "78": 10.01825, + "79": 10.0182, + "80": 10.03667, "81": 10.05422, - "82": 9.98253, - "83": 9.97442, - "84": 9.91747, - "85": 9.89499, + "82": 9.98252, + "83": 9.97441, + "84": 9.91745, + "85": 9.89498, "86": 9.99801, - "87": 9.99899, + "87": 9.99896, "88": 9.97974, - "89": 9.91874, + "89": 9.91873, "90": 9.91131, - "91": 9.94056, - "92": 9.92183, + "91": 9.94053, + "92": 9.92184, "93": 9.86312, "94": 9.93527, "95": 9.92344, - "96": 9.91216, - "97": 9.86241, - "98": 9.87102, - "99": 9.91079, + "96": 9.91215, + "97": 9.8624, + "98": 9.871, + "99": 9.91078, "100": 9.82083 } }, @@ -111,21 +111,21 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 568543232.0, + "1": 570640384.0, "2": 570640384.0, - "3": 568543232.0, + "3": 570640384.0, "4": 570640384.0, - "5": 568543232.0, + "5": 570640384.0, "6": 570640384.0, - "7": 568543232.0, + "7": 570640384.0, "8": 570640384.0, - "9": 568543232.0, + "9": 570640384.0, "10": 570640384.0, - "11": 568543232.0, + "11": 570640384.0, "12": 570640384.0, - "13": 568543232.0, + "13": 570640384.0, "14": 570640384.0, - "15": 568543232.0, + "15": 570640384.0, "16": 852351488.0, "17": 852351488.0, "18": 852351488.0, @@ -235,89 +235,89 @@ "15": 2393218048.0, "16": 2393218048.0, "17": 2675191296.0, - "18": 2675191296.0, - "19": 2675191296.0, - "20": 2675191296.0, - "21": 2675191296.0, - "22": 2675191296.0, - "23": 2675191296.0, - "24": 2675191296.0, - "25": 2675191296.0, - "26": 2675191296.0, - "27": 2675191296.0, - "28": 2675191296.0, - "29": 2675191296.0, - "30": 2675191296.0, - "31": 2675191296.0, - "32": 2675191296.0, - "33": 2675191296.0, - "34": 2675191296.0, - "35": 2675191296.0, - "36": 2675191296.0, - "37": 2675191296.0, - "38": 2675191296.0, - "39": 2675191296.0, - "40": 2675191296.0, - "41": 2675191296.0, - "42": 2675191296.0, - "43": 2675191296.0, - "44": 2675191296.0, - "45": 2675191296.0, - "46": 2675191296.0, - "47": 2675191296.0, - "48": 2675191296.0, - "49": 2675191296.0, - "50": 2675191296.0, - "51": 2675191296.0, - "52": 2675191296.0, - "53": 2675191296.0, - "54": 2675191296.0, - "55": 2675191296.0, - "56": 2675191296.0, - "57": 2675191296.0, - "58": 2675191296.0, - "59": 2675191296.0, - "60": 2675191296.0, - "61": 2675191296.0, - "62": 2675191296.0, - "63": 2675191296.0, - "64": 2675191296.0, - "65": 2675191296.0, - "66": 2675191296.0, - "67": 2675191296.0, - "68": 2675191296.0, - "69": 2675191296.0, - "70": 2675191296.0, - "71": 2675191296.0, - "72": 2675191296.0, - "73": 2675191296.0, - "74": 2675191296.0, - "75": 2675191296.0, - "76": 2675191296.0, - "77": 2675191296.0, - "78": 2675191296.0, - "79": 2675191296.0, - "80": 2675191296.0, - "81": 2675191296.0, - "82": 2675191296.0, - "83": 2675191296.0, - "84": 2675191296.0, - "85": 2675191296.0, - "86": 2675191296.0, - "87": 2675191296.0, - "88": 2675191296.0, - "89": 2675191296.0, - "90": 2675191296.0, - "91": 2675191296.0, - "92": 2675191296.0, - "93": 2675191296.0, - "94": 2675191296.0, - "95": 2675191296.0, - "96": 2675191296.0, - "97": 2675191296.0, - "98": 2675191296.0, - "99": 2675191296.0, - "100": 2675191296.0 + "18": 2676108288.0, + "19": 2676108288.0, + "20": 2676108288.0, + "21": 2676108288.0, + "22": 2676108288.0, + "23": 2676108288.0, + "24": 2676108288.0, + "25": 2676108288.0, + "26": 2676108288.0, + "27": 2676108288.0, + "28": 2676108288.0, + "29": 2676108288.0, + "30": 2676108288.0, + "31": 2676108288.0, + "32": 2676108288.0, + "33": 2676108288.0, + "34": 2676108288.0, + "35": 2676108288.0, + "36": 2676108288.0, + "37": 2676108288.0, + "38": 2676108288.0, + "39": 2676108288.0, + "40": 2676108288.0, + "41": 2676108288.0, + "42": 2676108288.0, + "43": 2676108288.0, + "44": 2676108288.0, + "45": 2676108288.0, + "46": 2676108288.0, + "47": 2676108288.0, + "48": 2676108288.0, + "49": 2676108288.0, + "50": 2676108288.0, + "51": 2676108288.0, + "52": 2676108288.0, + "53": 2676108288.0, + "54": 2676108288.0, + "55": 2676108288.0, + "56": 2676108288.0, + "57": 2676108288.0, + "58": 2676108288.0, + "59": 2676108288.0, + "60": 2676108288.0, + "61": 2676108288.0, + "62": 2676108288.0, + "63": 2676108288.0, + "64": 2676108288.0, + "65": 2676108288.0, + "66": 2676108288.0, + "67": 2676108288.0, + "68": 2676108288.0, + "69": 2676108288.0, + "70": 2676108288.0, + "71": 2676108288.0, + "72": 2676108288.0, + "73": 2676108288.0, + "74": 2676108288.0, + "75": 2676108288.0, + "76": 2676108288.0, + "77": 2676108288.0, + "78": 2676108288.0, + "79": 2676108288.0, + "80": 2676108288.0, + "81": 2676108288.0, + "82": 2676108288.0, + "83": 2676108288.0, + "84": 2676108288.0, + "85": 2676108288.0, + "86": 2676108288.0, + "87": 2676108288.0, + "88": 2676108288.0, + "89": 2676108288.0, + "90": 2676108288.0, + "91": 2676108288.0, + "92": 2676108288.0, + "93": 2676108288.0, + "94": 2676108288.0, + "95": 2676108288.0, + "96": 2676108288.0, + "97": 2676108288.0, + "98": 2676108288.0, + "99": 2676108288.0, + "100": 2676108288.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.83496, - "3": 0.16853, - "4": 0.14306, - "5": 0.14329, - "6": 0.14329, - "7": 0.14329, - "8": 0.14416, - "9": 0.14363, - "10": 0.14405, - "11": 0.1433, - "12": 0.14358, - "13": 0.14369, - "14": 0.14339, - "15": 0.14346, - "16": 0.20571, - "17": 0.15341, - "18": 0.14809, - "19": 0.14805, - "20": 0.14887, - "21": 0.14782, - "22": 0.14839, - "23": 0.14855, - "24": 0.14779, - "25": 0.14766, - "26": 0.14786, - "27": 0.14779, - "28": 0.14787, - "29": 0.14757, - "30": 0.14788, - "31": 0.1474, - "32": 0.14301, - "33": 0.14777, - "34": 0.1475, - "35": 0.1483, - "36": 0.14826, - "37": 0.14852, - "38": 0.1478, - "39": 0.14793, - "40": 0.14777, - "41": 0.14807, - "42": 0.14779, - "43": 0.14808, - "44": 0.14766, - "45": 0.14801, - "46": 0.14781, - "47": 0.14849, - "48": 0.14867, - "49": 0.14699, - "50": 0.14693, - "51": 0.47783, - "52": 0.18401, - "53": 0.15249, - "54": 0.15225, - "55": 0.15095, - "56": 0.15124, - "57": 0.15027, - "58": 0.15038, - "59": 0.15055, - "60": 0.15002, - "61": 0.14984, - "62": 0.15019, - "63": 0.15011, - "64": 0.14975, - "65": 0.14897, - "66": 0.14992, - "67": 0.14939, - "68": 0.15019, - "69": 0.14823, - "70": 0.14873, - "71": 0.14856, - "72": 0.14794, - "73": 0.1476, - "74": 0.14777, - "75": 0.1481, - "76": 0.14895, - "77": 0.14745, - "78": 0.14937, - "79": 0.14804, - "80": 0.14816, - "81": 0.14838, - "82": 0.1486, - "83": 0.14795, - "84": 0.14762, - "85": 0.14796, - "86": 0.14764, - "87": 0.14768, - "88": 0.14781, - "89": 0.14791, - "90": 0.14732, - "91": 0.14766, - "92": 0.14766, - "93": 0.14752, - "94": 0.14777, - "95": 0.14741, - "96": 0.14771, - "97": 0.14804, - "98": 0.14778, - "99": 0.14823, - "100": 0.14749 + "2": 5.14474, + "3": 0.16987, + "4": 0.14883, + "5": 0.1489, + "6": 0.14887, + "7": 0.14866, + "8": 0.1486, + "9": 0.15113, + "10": 0.15083, + "11": 0.1502, + "12": 0.15165, + "13": 0.14947, + "14": 0.14961, + "15": 0.1497, + "16": 0.21119, + "17": 0.16063, + "18": 0.15442, + "19": 0.15292, + "20": 0.1536, + "21": 0.15297, + "22": 0.15308, + "23": 0.15381, + "24": 0.1532, + "25": 0.15384, + "26": 0.15337, + "27": 0.15388, + "28": 0.15311, + "29": 0.15277, + "30": 0.15569, + "31": 0.15525, + "32": 0.15041, + "33": 0.15564, + "34": 0.15505, + "35": 0.15563, + "36": 0.15545, + "37": 0.15495, + "38": 0.15529, + "39": 0.15573, + "40": 0.15574, + "41": 0.15571, + "42": 0.15603, + "43": 0.15493, + "44": 0.15566, + "45": 0.15766, + "46": 0.15722, + "47": 0.1568, + "48": 0.15766, + "49": 0.15916, + "50": 0.15655, + "51": 0.37666, + "52": 0.19478, + "53": 0.15633, + "54": 0.15533, + "55": 0.1557, + "56": 0.15434, + "57": 0.15381, + "58": 0.15407, + "59": 0.15374, + "60": 0.15347, + "61": 0.15386, + "62": 0.15385, + "63": 0.15379, + "64": 0.15355, + "65": 0.15386, + "66": 0.15372, + "67": 0.15413, + "68": 0.15375, + "69": 0.15379, + "70": 0.15424, + "71": 0.15369, + "72": 0.15383, + "73": 0.1536, + "74": 0.15346, + "75": 0.15398, + "76": 0.15383, + "77": 0.15418, + "78": 0.15335, + "79": 0.15387, + "80": 0.15328, + "81": 0.15326, + "82": 0.15319, + "83": 0.15249, + "84": 0.15304, + "85": 0.15286, + "86": 0.1528, + "87": 0.15267, + "88": 0.15294, + "89": 0.1543, + "90": 0.15285, + "91": 0.15213, + "92": 0.15779, + "93": 0.15306, + "94": 0.15278, + "95": 0.15298, + "96": 0.15314, + "97": 0.15255, + "98": 0.15261, + "99": 0.15342, + "100": 0.15217 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2113.0, - "17": 2560.0, - "18": 2507.0, - "19": 2072.0, - "20": 2394.0, - "21": 2415.0, - "22": 2187.0, - "23": 2111.0, - "24": 2355.0, - "25": 2174.0, - "26": 2219.0, - "27": 2369.0, - "28": 2361.0, - "29": 2430.0, - "30": 2200.0, - "31": 2607.0, + "16": 2092.0, + "17": 2534.0, + "18": 2459.0, + "19": 2122.0, + "20": 2523.0, + "21": 2320.0, + "22": 2371.0, + "23": 2094.0, + "24": 2359.0, + "25": 2078.0, + "26": 2255.0, + "27": 2398.0, + "28": 2425.0, + "29": 2413.0, + "30": 2105.0, + "31": 2575.0, "32": "nan", - "33": 2327.0, - "34": 2495.0, - "35": 2607.0, - "36": 2220.0, - "37": 2443.0, - "38": 2491.0, - "39": 2448.0, - "40": 2335.0, + "33": 2375.0, + "34": 2519.0, + "35": 2604.0, + "36": 2286.0, + "37": 2314.0, + "38": 2634.0, + "39": 2393.0, + "40": 2348.0, "41": 2387.0, - "42": 2147.0, - "43": 2583.0, - "44": 2113.0, - "45": 2776.0, - "46": 2342.0, - "47": 2360.0, - "48": 2502.0, - "49": 2663.0, - "50": 2499.0, - "51": 2379.0, - "52": 2642.0, - "53": 2666.0, - "54": 2342.0, - "55": 2417.0, - "56": 2512.0, - "57": 2374.0, - "58": 3193.0, - "59": 2612.0, - "60": 3054.0, - "61": 2479.0, - "62": 3005.0, - "63": 3298.0, - "64": 2905.0, - "65": 2931.0, - "66": 3060.0, - "67": 3200.0, - "68": 3304.0, - "69": 2958.0, - "70": 3483.0, - "71": 2858.0, - "72": 2889.0, - "73": 3275.0, - "74": 3202.0, - "75": 4067.0, - "76": 3711.0, - "77": 3488.0, - "78": 3565.0, - "79": 5231.0, - "80": 3713.0, - "81": 4025.0, - "82": 3101.0, - "83": 3403.0, - "84": 4214.0, - "85": 4343.0, - "86": 3566.0, - "87": 2688.0, - "88": 4116.0, - "89": 4408.0, - "90": 4993.0, - "91": 2805.0, - "92": 4068.0, - "93": 4315.0, - "94": 3166.0, - "95": 3693.0, - "96": 4850.0, - "97": 3658.0, - "98": 4301.0, - "99": 3799.0, - "100": 3576.0 + "42": 2222.0, + "43": 2529.0, + "44": 2253.0, + "45": 2821.0, + "46": 2370.0, + "47": 2408.0, + "48": 2492.0, + "49": 2545.0, + "50": 2614.0, + "51": 2533.0, + "52": 2595.0, + "53": 2656.0, + "54": 2402.0, + "55": 2425.0, + "56": 2560.0, + "57": 2417.0, + "58": 3289.0, + "59": 2614.0, + "60": 2996.0, + "61": 2482.0, + "62": 3087.0, + "63": 3286.0, + "64": 2901.0, + "65": 3013.0, + "66": 3061.0, + "67": 3241.0, + "68": 3179.0, + "69": 2954.0, + "70": 3439.0, + "71": 2881.0, + "72": 2916.0, + "73": 3266.0, + "74": 3249.0, + "75": 4049.0, + "76": 3806.0, + "77": 3608.0, + "78": 3591.0, + "79": 5194.0, + "80": 3686.0, + "81": 4099.0, + "82": 2999.0, + "83": 3369.0, + "84": 4240.0, + "85": 4359.0, + "86": 3570.0, + "87": 2691.0, + "88": 4219.0, + "89": 4277.0, + "90": 4963.0, + "91": 2847.0, + "92": 3990.0, + "93": 4151.0, + "94": 3190.0, + "95": 3824.0, + "96": 4701.0, + "97": 3697.0, + "98": 4229.0, + "99": 3807.0, + "100": 3463.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json index 1b3013fa07b..52ec56c26d8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9275, - "2": 10.92325, - "3": 10.93202, - "4": 10.93451, + "1": 10.92746, + "2": 10.92322, + "3": 10.93207, + "4": 10.93449, "5": 10.93111, - "6": 10.92354, - "7": 10.93564, - "8": 10.92691, - "9": 10.93221, + "6": 10.92351, + "7": 10.93565, + "8": 10.92689, + "9": 10.9322, "10": 10.93335, "11": 10.92908, - "12": 10.93838, - "13": 10.93346, - "14": 10.92436, - "15": 10.92678, + "12": 10.93839, + "13": 10.93345, + "14": 10.92438, + "15": 10.92675, "16": 10.9266, - "17": 10.93771, - "18": 10.92511, - "19": 10.93139, - "20": 10.911, + "17": 10.93766, + "18": 10.92513, + "19": 10.9314, + "20": 10.91103, "21": 10.92839, - "22": 10.93041, - "23": 10.92327, - "24": 10.90331, - "25": 10.90996, - "26": 10.90216, - "27": 10.88423, - "28": 10.87039, - "29": 10.86343, + "22": 10.93044, + "23": 10.92328, + "24": 10.90332, + "25": 10.90992, + "26": 10.90213, + "27": 10.88424, + "28": 10.87041, + "29": 10.86345, "30": 10.84861, - "31": 10.84245, - "32": 10.81777, - "33": 10.80498, - "34": 10.76474, - "35": 10.77131, - "36": 10.74516, + "31": 10.84247, + "32": 10.81773, + "33": 10.805, + "34": 10.76472, + "35": 10.77129, + "36": 10.74517, "37": 10.72331, - "38": 10.7054, - "39": 10.67725, - "40": 10.66445, - "41": 10.63456, - "42": 10.61595, - "43": 10.59723, + "38": 10.70539, + "39": 10.67722, + "40": 10.66444, + "41": 10.63455, + "42": 10.61594, + "43": 10.59722, "44": 10.56907, - "45": 10.57338, - "46": 10.53669, - "47": 10.50713, + "45": 10.57334, + "46": 10.5367, + "47": 10.50712, "48": 10.47164, - "49": 10.45126, - "50": 10.45014, - "51": 10.43031, - "52": 10.39165, + "49": 10.45125, + "50": 10.45015, + "51": 10.43026, + "52": 10.39166, "53": 10.3893, - "54": 10.35084, - "55": 10.315, - "56": 10.33409, - "57": 10.3121, - "58": 10.31358, + "54": 10.35081, + "55": 10.31496, + "56": 10.33408, + "57": 10.31209, + "58": 10.31359, "59": 10.26031, "60": 10.26798, - "61": 10.22337, - "62": 10.18262, - "63": 10.23312, - "64": 10.18919, - "65": 10.16315, - "66": 10.18066, - "67": 10.14893, - "68": 10.11197, - "69": 10.13061, - "70": 10.1092, - "71": 10.11532, - "72": 10.10067, + "61": 10.22336, + "62": 10.18264, + "63": 10.23309, + "64": 10.1892, + "65": 10.16314, + "66": 10.18064, + "67": 10.1489, + "68": 10.11196, + "69": 10.1306, + "70": 10.10917, + "71": 10.1153, + "72": 10.10064, "73": 10.09173, - "74": 10.07986, - "75": 10.04376, - "76": 10.07378, - "77": 10.06591, + "74": 10.07985, + "75": 10.04375, + "76": 10.07377, + "77": 10.06593, "78": 10.01925, - "79": 10.03038, - "80": 10.04608, - "81": 10.05931, - "82": 9.99946, + "79": 10.03036, + "80": 10.04606, + "81": 10.0593, + "82": 9.99944, "83": 9.97676, - "84": 9.91162, - "85": 9.9072, + "84": 9.91163, + "85": 9.90718, "86": 9.99655, "87": 10.01053, "88": 9.99368, - "89": 9.92505, + "89": 9.92504, "90": 9.9196, - "91": 9.94844, + "91": 9.94842, "92": 9.92279, - "93": 9.86188, + "93": 9.86187, "94": 9.93555, - "95": 9.92702, + "95": 9.92701, "96": 9.90567, - "97": 9.85584, - "98": 9.87531, - "99": 9.91439, - "100": 9.81824 + "97": 9.85582, + "98": 9.87528, + "99": 9.91437, + "100": 9.81822 } }, "mem-allocated-bytes": { @@ -126,91 +126,91 @@ "13": 569591808.0, "14": 569591808.0, "15": 569591808.0, - "16": 852351488.0, - "17": 852351488.0, - "18": 852351488.0, - "19": 852351488.0, - "20": 852351488.0, - "21": 852351488.0, - "22": 852351488.0, - "23": 852351488.0, - "24": 852351488.0, - "25": 852351488.0, - "26": 852351488.0, - "27": 852351488.0, - "28": 852351488.0, - "29": 852351488.0, - "30": 852351488.0, - "31": 852351488.0, - "32": 852351488.0, - "33": 852351488.0, - "34": 852351488.0, - "35": 852351488.0, - "36": 852351488.0, - "37": 852351488.0, - "38": 852351488.0, - "39": 852351488.0, - "40": 852351488.0, - "41": 852351488.0, - "42": 852351488.0, - "43": 852351488.0, - "44": 852351488.0, - "45": 852351488.0, - "46": 852351488.0, - "47": 852351488.0, - "48": 852351488.0, - "49": 852351488.0, - "50": 852351488.0, - "51": 852351488.0, - "52": 852351488.0, - "53": 852351488.0, - "54": 852351488.0, - "55": 852351488.0, - "56": 852351488.0, - "57": 852351488.0, - "58": 852351488.0, - "59": 852351488.0, - "60": 852351488.0, - "61": 852351488.0, - "62": 852351488.0, - "63": 852351488.0, - "64": 852351488.0, - "65": 852351488.0, - "66": 852351488.0, - "67": 852351488.0, - "68": 852351488.0, - "69": 852351488.0, - "70": 852351488.0, - "71": 852351488.0, - "72": 852351488.0, - "73": 852351488.0, - "74": 852351488.0, - "75": 852351488.0, - "76": 852351488.0, - "77": 852351488.0, - "78": 852351488.0, - "79": 852351488.0, - "80": 852351488.0, - "81": 852351488.0, - "82": 852351488.0, - "83": 852351488.0, - "84": 852351488.0, - "85": 852351488.0, - "86": 852351488.0, - "87": 852351488.0, - "88": 852351488.0, - "89": 852351488.0, - "90": 852351488.0, - "91": 852351488.0, - "92": 852351488.0, - "93": 852351488.0, - "94": 852351488.0, - "95": 852351488.0, - "96": 852351488.0, - "97": 852351488.0, - "98": 852351488.0, - "99": 852351488.0, - "100": 852351488.0 + "16": 851302912.0, + "17": 851302912.0, + "18": 851302912.0, + "19": 851302912.0, + "20": 851302912.0, + "21": 851302912.0, + "22": 851302912.0, + "23": 851302912.0, + "24": 851302912.0, + "25": 851302912.0, + "26": 851302912.0, + "27": 851302912.0, + "28": 851302912.0, + "29": 851302912.0, + "30": 851302912.0, + "31": 851302912.0, + "32": 851302912.0, + "33": 851302912.0, + "34": 851302912.0, + "35": 851302912.0, + "36": 851302912.0, + "37": 851302912.0, + "38": 851302912.0, + "39": 851302912.0, + "40": 851302912.0, + "41": 851302912.0, + "42": 851302912.0, + "43": 851302912.0, + "44": 851302912.0, + "45": 851302912.0, + "46": 851302912.0, + "47": 851302912.0, + "48": 851302912.0, + "49": 851302912.0, + "50": 851302912.0, + "51": 851302912.0, + "52": 851302912.0, + "53": 851302912.0, + "54": 851302912.0, + "55": 851302912.0, + "56": 851302912.0, + "57": 851302912.0, + "58": 851302912.0, + "59": 851302912.0, + "60": 851302912.0, + "61": 851302912.0, + "62": 851302912.0, + "63": 851302912.0, + "64": 851302912.0, + "65": 851302912.0, + "66": 851302912.0, + "67": 851302912.0, + "68": 851302912.0, + "69": 851302912.0, + "70": 851302912.0, + "71": 851302912.0, + "72": 851302912.0, + "73": 851302912.0, + "74": 851302912.0, + "75": 851302912.0, + "76": 851302912.0, + "77": 851302912.0, + "78": 851302912.0, + "79": 851302912.0, + "80": 851302912.0, + "81": 851302912.0, + "82": 851302912.0, + "83": 851302912.0, + "84": 851302912.0, + "85": 851302912.0, + "86": 851302912.0, + "87": 851302912.0, + "88": 851302912.0, + "89": 851302912.0, + "90": 851302912.0, + "91": 851302912.0, + "92": 851302912.0, + "93": 851302912.0, + "94": 851302912.0, + "95": 851302912.0, + "96": 851302912.0, + "97": 851302912.0, + "98": 851302912.0, + "99": 851302912.0, + "100": 851302912.0 } }, "mem-max-allocated-bytes": { @@ -234,90 +234,90 @@ "14": 2394266624.0, "15": 2394266624.0, "16": 2394266624.0, - "17": 2677288448.0, - "18": 2677288448.0, - "19": 2677288448.0, - "20": 2677288448.0, - "21": 2677288448.0, - "22": 2677288448.0, - "23": 2677288448.0, - "24": 2677288448.0, - "25": 2677288448.0, - "26": 2677288448.0, - "27": 2677288448.0, - "28": 2677288448.0, - "29": 2677288448.0, - "30": 2677288448.0, - "31": 2677288448.0, - "32": 2677288448.0, - "33": 2677288448.0, - "34": 2677288448.0, - "35": 2677288448.0, - "36": 2677288448.0, - "37": 2677288448.0, - "38": 2677288448.0, - "39": 2677288448.0, - "40": 2677288448.0, - "41": 2677288448.0, - "42": 2677288448.0, - "43": 2677288448.0, - "44": 2677288448.0, - "45": 2677288448.0, - "46": 2677288448.0, - "47": 2677288448.0, - "48": 2677288448.0, - "49": 2677288448.0, - "50": 2677288448.0, - "51": 2677288448.0, - "52": 2677288448.0, - "53": 2677288448.0, - "54": 2677288448.0, - "55": 2677288448.0, - "56": 2677288448.0, - "57": 2677288448.0, - "58": 2677288448.0, - "59": 2677288448.0, - "60": 2677288448.0, - "61": 2677288448.0, - "62": 2677288448.0, - "63": 2677288448.0, - "64": 2677288448.0, - "65": 2677288448.0, - "66": 2677288448.0, - "67": 2677288448.0, - "68": 2677288448.0, - "69": 2677288448.0, - "70": 2677288448.0, - "71": 2677288448.0, - "72": 2677288448.0, - "73": 2677288448.0, - "74": 2677288448.0, - "75": 2677288448.0, - "76": 2677288448.0, - "77": 2677288448.0, - "78": 2677288448.0, - "79": 2677288448.0, - "80": 2677288448.0, - "81": 2677288448.0, - "82": 2677288448.0, - "83": 2677288448.0, - "84": 2677288448.0, - "85": 2677288448.0, - "86": 2677288448.0, - "87": 2677288448.0, - "88": 2677288448.0, - "89": 2677288448.0, - "90": 2677288448.0, - "91": 2677288448.0, - "92": 2677288448.0, - "93": 2677288448.0, - "94": 2677288448.0, - "95": 2677288448.0, - "96": 2677288448.0, - "97": 2677288448.0, - "98": 2677288448.0, - "99": 2677288448.0, - "100": 2677288448.0 + "17": 2675977728.0, + "18": 2675977728.0, + "19": 2675977728.0, + "20": 2675977728.0, + "21": 2675977728.0, + "22": 2675977728.0, + "23": 2675977728.0, + "24": 2675977728.0, + "25": 2675977728.0, + "26": 2675977728.0, + "27": 2675977728.0, + "28": 2675977728.0, + "29": 2675977728.0, + "30": 2675977728.0, + "31": 2675977728.0, + "32": 2675977728.0, + "33": 2675977728.0, + "34": 2675977728.0, + "35": 2675977728.0, + "36": 2675977728.0, + "37": 2675977728.0, + "38": 2675977728.0, + "39": 2675977728.0, + "40": 2675977728.0, + "41": 2675977728.0, + "42": 2675977728.0, + "43": 2675977728.0, + "44": 2675977728.0, + "45": 2675977728.0, + "46": 2675977728.0, + "47": 2675977728.0, + "48": 2675977728.0, + "49": 2675977728.0, + "50": 2675977728.0, + "51": 2675977728.0, + "52": 2675977728.0, + "53": 2675977728.0, + "54": 2675977728.0, + "55": 2675977728.0, + "56": 2675977728.0, + "57": 2675977728.0, + "58": 2675977728.0, + "59": 2675977728.0, + "60": 2675977728.0, + "61": 2675977728.0, + "62": 2675977728.0, + "63": 2675977728.0, + "64": 2675977728.0, + "65": 2675977728.0, + "66": 2675977728.0, + "67": 2675977728.0, + "68": 2675977728.0, + "69": 2675977728.0, + "70": 2675977728.0, + "71": 2675977728.0, + "72": 2675977728.0, + "73": 2675977728.0, + "74": 2675977728.0, + "75": 2675977728.0, + "76": 2675977728.0, + "77": 2675977728.0, + "78": 2675977728.0, + "79": 2675977728.0, + "80": 2675977728.0, + "81": 2675977728.0, + "82": 2675977728.0, + "83": 2675977728.0, + "84": 2675977728.0, + "85": 2675977728.0, + "86": 2675977728.0, + "87": 2675977728.0, + "88": 2675977728.0, + "89": 2675977728.0, + "90": 2675977728.0, + "91": 2675977728.0, + "92": 2675977728.0, + "93": 2675977728.0, + "94": 2675977728.0, + "95": 2675977728.0, + "96": 2675977728.0, + "97": 2675977728.0, + "98": 2675977728.0, + "99": 2675977728.0, + "100": 2675977728.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.38812, - "3": 0.16001, - "4": 0.13803, - "5": 0.13648, - "6": 0.13593, - "7": 0.13689, - "8": 0.13712, - "9": 0.13744, - "10": 0.13599, - "11": 0.13728, - "12": 0.13775, - "13": 0.13792, - "14": 0.13672, - "15": 0.13842, - "16": 0.21203, - "17": 0.15409, - "18": 0.14014, - "19": 0.14345, - "20": 0.14446, - "21": 0.14495, - "22": 0.1453, - "23": 0.14496, - "24": 0.14362, - "25": 0.14525, - "26": 0.14445, - "27": 0.14548, - "28": 0.14476, - "29": 0.14548, - "30": 0.14542, - "31": 0.14339, - "32": 0.14417, - "33": 0.14273, - "34": 0.14383, - "35": 0.1458, - "36": 0.1432, - "37": 0.14459, - "38": 0.14471, - "39": 0.1453, - "40": 0.14531, - "41": 0.14578, - "42": 0.14402, - "43": 0.14442, - "44": 0.14394, - "45": 0.14481, - "46": 0.14444, - "47": 0.14493, - "48": 0.14513, - "49": 0.14532, - "50": 0.14575, - "51": 0.30107, - "52": 0.15372, - "53": 0.17263, - "54": 0.14308, - "55": 0.14404, - "56": 0.14437, - "57": 0.14458, - "58": 0.14355, - "59": 0.14447, - "60": 0.14456, - "61": 0.14477, - "62": 0.14541, - "63": 0.14405, - "64": 0.14307, - "65": 0.14449, - "66": 0.14355, - "67": 0.14442, - "68": 0.14453, - "69": 0.14483, - "70": 0.14475, - "71": 0.14493, - "72": 0.14441, - "73": 0.14327, - "74": 0.14442, - "75": 0.1464, - "76": 0.147, - "77": 0.14463, - "78": 0.1436, - "79": 0.1447, - "80": 0.14536, - "81": 0.14488, - "82": 0.1462, - "83": 0.14575, - "84": 0.14566, - "85": 0.14515, - "86": 0.14533, - "87": 0.14532, - "88": 0.144, - "89": 0.14562, - "90": 0.14602, - "91": 0.14601, - "92": 0.14671, - "93": 0.1475, - "94": 0.14463, - "95": 0.14355, - "96": 0.14483, - "97": 0.14497, - "98": 0.14508, - "99": 0.14496, - "100": 0.14692 + "2": 8.17646, + "3": 0.16138, + "4": 0.11538, + "5": 0.11613, + "6": 0.11412, + "7": 0.23488, + "8": 0.1156, + "9": 0.11522, + "10": 0.11346, + "11": 0.11562, + "12": 0.11383, + "13": 0.11348, + "14": 0.11423, + "15": 0.25152, + "16": 0.18768, + "17": 0.13017, + "18": 0.11566, + "19": 0.11845, + "20": 0.11882, + "21": 0.11711, + "22": 0.24392, + "23": 0.11725, + "24": 0.11778, + "25": 0.12093, + "26": 0.11847, + "27": 0.11835, + "28": 0.11847, + "29": 0.11723, + "30": 0.11673, + "31": 0.11793, + "32": 0.15361, + "33": 0.12012, + "34": 0.11692, + "35": 0.11861, + "36": 0.20875, + "37": 0.11907, + "38": 0.1177, + "39": 0.11911, + "40": 0.11791, + "41": 0.11804, + "42": 0.11623, + "43": 0.25299, + "44": 0.1193, + "45": 0.1195, + "46": 0.24165, + "47": 0.11925, + "48": 0.11897, + "49": 0.22722, + "50": 0.11896, + "51": 0.27976, + "52": 0.16731, + "53": 0.12243, + "54": 0.11933, + "55": 0.11943, + "56": 0.11759, + "57": 0.11794, + "58": 0.11568, + "59": 0.11786, + "60": 0.11809, + "61": 0.11659, + "62": 0.11856, + "63": 0.11783, + "64": 0.11935, + "65": 0.11889, + "66": 0.11786, + "67": 0.1174, + "68": 0.11815, + "69": 0.11816, + "70": 0.11806, + "71": 0.11801, + "72": 0.12012, + "73": 0.11875, + "74": 0.11801, + "75": 0.11736, + "76": 0.11877, + "77": 0.11774, + "78": 0.12005, + "79": 0.11914, + "80": 0.11926, + "81": 0.12147, + "82": 0.11784, + "83": 0.11636, + "84": 0.11754, + "85": 0.11922, + "86": 0.12008, + "87": 0.11813, + "88": 0.11929, + "89": 0.11853, + "90": 0.11657, + "91": 0.11827, + "92": 0.11972, + "93": 0.11979, + "94": 0.12032, + "95": 0.11955, + "96": 0.11858, + "97": 0.12031, + "98": 0.24002, + "99": 0.12121, + "100": 0.12071 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2200.0, - "17": 2776.0, + "16": 2238.0, + "17": 2736.0, "18": "nan", - "19": 2199.0, - "20": 2482.0, - "21": 2496.0, - "22": 2214.0, - "23": 2186.0, - "24": 2405.0, - "25": 2076.0, - "26": 2423.0, - "27": 2287.0, - "28": 2422.0, - "29": 2500.0, - "30": 2082.0, - "31": 2630.0, - "32": 2364.0, - "33": 2242.0, - "34": 2465.0, - "35": 2434.0, - "36": 2304.0, - "37": 2427.0, - "38": 2583.0, - "39": 2498.0, - "40": 2326.0, - "41": 2381.0, - "42": 2272.0, - "43": 2647.0, - "44": 2301.0, - "45": 2878.0, - "46": 2232.0, - "47": 2368.0, - "48": 2280.0, - "49": 2514.0, - "50": 2476.0, - "51": 2281.0, - "52": 2734.0, - "53": 2710.0, - "54": 2389.0, - "55": 2377.0, - "56": 2753.0, - "57": 2289.0, - "58": 3201.0, - "59": 2642.0, - "60": 2937.0, - "61": 2377.0, - "62": 3072.0, - "63": 3104.0, - "64": 2918.0, - "65": 2894.0, - "66": 3147.0, - "67": 3149.0, - "68": 3325.0, - "69": 2874.0, - "70": 3453.0, - "71": 2982.0, - "72": 3032.0, - "73": 3268.0, - "74": 3136.0, - "75": 3911.0, - "76": 3473.0, - "77": 3447.0, - "78": 3581.0, - "79": 5169.0, - "80": 3770.0, - "81": 3813.0, - "82": 3032.0, - "83": 3491.0, - "84": 3950.0, - "85": 3977.0, - "86": 3426.0, - "87": 2734.0, - "88": 4419.0, - "89": 4190.0, - "90": 4837.0, - "91": 2979.0, - "92": 3827.0, - "93": 3990.0, - "94": 3346.0, - "95": 3785.0, - "96": 4396.0, - "97": 4423.0, - "98": 4592.0, - "99": 3667.0, - "100": 4011.0 + "19": 2148.0, + "20": 2479.0, + "21": 2433.0, + "22": 2180.0, + "23": 2202.0, + "24": 2394.0, + "25": 2110.0, + "26": 2372.0, + "27": 2388.0, + "28": 2395.0, + "29": 2476.0, + "30": 2121.0, + "31": 2576.0, + "32": 2316.0, + "33": 2265.0, + "34": 2589.0, + "35": 2438.0, + "36": 2279.0, + "37": 2338.0, + "38": 2595.0, + "39": 2509.0, + "40": 2360.0, + "41": 2375.0, + "42": 2258.0, + "43": 2665.0, + "44": 2349.0, + "45": 2901.0, + "46": 2320.0, + "47": 2310.0, + "48": 2492.0, + "49": 2590.0, + "50": 2561.0, + "51": 2366.0, + "52": 2612.0, + "53": 2687.0, + "54": 2330.0, + "55": 2425.0, + "56": 2653.0, + "57": 2380.0, + "58": 3310.0, + "59": 2521.0, + "60": 2955.0, + "61": 2497.0, + "62": 3121.0, + "63": 3247.0, + "64": 2932.0, + "65": 2883.0, + "66": 3154.0, + "67": 3219.0, + "68": 3264.0, + "69": 2776.0, + "70": 3493.0, + "71": 3036.0, + "72": 2934.0, + "73": 3236.0, + "74": 3091.0, + "75": 3957.0, + "76": 3609.0, + "77": 3471.0, + "78": 3680.0, + "79": 5199.0, + "80": 3800.0, + "81": 3839.0, + "82": 2968.0, + "83": 3437.0, + "84": 3978.0, + "85": 3925.0, + "86": 3440.0, + "87": 2620.0, + "88": 4446.0, + "89": 4233.0, + "90": 4742.0, + "91": 3057.0, + "92": 3784.0, + "93": 3912.0, + "94": 3373.0, + "95": 3770.0, + "96": 4373.0, + "97": 4495.0, + "98": 4570.0, + "99": 3484.0, + "100": 3977.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json index 155c5a98068..58ab65374bd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89629, - "2": 10.88412, + "1": 10.89631, + "2": 10.88413, "3": 10.88476, "4": 10.88014, - "5": 10.8825, - "6": 10.87881, - "7": 10.8837, - "8": 10.88065, - "9": 10.88602, + "5": 10.88249, + "6": 10.8788, + "7": 10.88368, + "8": 10.88067, + "9": 10.88608, "10": 10.88235, - "11": 10.88644, - "12": 10.88403, - "13": 10.88226, - "14": 10.88319, - "15": 10.8795, - "16": 10.88084, + "11": 10.88646, + "12": 10.884, + "13": 10.88224, + "14": 10.88321, + "15": 10.87947, + "16": 10.8808, "17": 10.89482, "18": 10.87831, - "19": 10.89758, - "20": 10.87963, - "21": 10.88892, - "22": 10.88443, + "19": 10.89755, + "20": 10.87959, + "21": 10.88893, + "22": 10.88442, "23": 10.87227, - "24": 10.85702, + "24": 10.85705, "25": 10.86099, - "26": 10.83645, - "27": 10.83445, - "28": 10.83039, - "29": 10.8082, - "30": 10.78668, - "31": 10.77883, + "26": 10.83647, + "27": 10.83442, + "28": 10.83038, + "29": 10.80813, + "30": 10.78669, + "31": 10.7788, "32": 10.77361, "33": 10.74029, - "34": 10.72355, + "34": 10.72353, "35": 10.71304, - "36": 10.68845, - "37": 10.66097, + "36": 10.6884, + "37": 10.66094, "38": 10.65601, - "39": 10.63174, - "40": 10.61698, - "41": 10.58755, - "42": 10.5645, - "43": 10.54922, + "39": 10.63172, + "40": 10.61696, + "41": 10.58756, + "42": 10.56449, + "43": 10.54924, "44": 10.52112, - "45": 10.51181, - "46": 10.48416, - "47": 10.4652, + "45": 10.5118, + "46": 10.48418, + "47": 10.46518, "48": 10.42505, - "49": 10.41707, - "50": 10.40407, - "51": 10.39841, + "49": 10.41706, + "50": 10.40404, + "51": 10.39839, "52": 10.35696, - "53": 10.35712, - "54": 10.31836, - "55": 10.29143, - "56": 10.29525, - "57": 10.28175, - "58": 10.28576, - "59": 10.23425, - "60": 10.23902, - "61": 10.18539, - "62": 10.15955, - "63": 10.21087, - "64": 10.17133, - "65": 10.14708, + "53": 10.35711, + "54": 10.31833, + "55": 10.29141, + "56": 10.29526, + "57": 10.28173, + "58": 10.28575, + "59": 10.23424, + "60": 10.239, + "61": 10.18538, + "62": 10.15953, + "63": 10.21086, + "64": 10.17131, + "65": 10.14707, "66": 10.15528, - "67": 10.13213, - "68": 10.09564, + "67": 10.13211, + "68": 10.09562, "69": 10.11243, - "70": 10.08369, - "71": 10.10364, - "72": 10.08994, - "73": 10.08361, + "70": 10.08366, + "71": 10.10363, + "72": 10.08995, + "73": 10.08359, "74": 10.0635, - "75": 10.04314, - "76": 10.06399, - "77": 10.05648, - "78": 10.01452, - "79": 10.02446, - "80": 10.04024, - "81": 10.05974, - "82": 9.99072, - "83": 9.97283, + "75": 10.04313, + "76": 10.064, + "77": 10.05647, + "78": 10.01453, + "79": 10.02445, + "80": 10.04023, + "81": 10.05971, + "82": 9.99073, + "83": 9.97282, "84": 9.90914, - "85": 9.91074, + "85": 9.91073, "86": 9.99152, - "87": 9.9971, + "87": 9.99709, "88": 9.98034, - "89": 9.92164, + "89": 9.92163, "90": 9.91678, - "91": 9.95049, - "92": 9.92022, - "93": 9.85907, - "94": 9.9275, - "95": 9.92325, - "96": 9.8979, - "97": 9.853, - "98": 9.87376, + "91": 9.95046, + "92": 9.92018, + "93": 9.85906, + "94": 9.92749, + "95": 9.92324, + "96": 9.89787, + "97": 9.85298, + "98": 9.87373, "99": 9.91362, - "100": 9.81531 + "100": 9.81529 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1465368064.0, - "2": 1465368576.0, - "3": 1465368576.0, - "4": 1465368576.0, - "5": 1465368576.0, - "6": 1465368576.0, - "7": 1465368576.0, - "8": 1465368576.0, - "9": 1465368576.0, - "10": 1465368576.0, - "11": 1465368576.0, - "12": 1465368576.0, - "13": 1465368576.0, - "14": 1465368576.0, - "15": 1465368576.0, - "16": 1465368576.0, - "17": 1597485568.0, - "18": 1597485568.0, - "19": 1597485568.0, - "20": 1597485568.0, - "21": 1597485568.0, - "22": 1597485568.0, - "23": 1597485568.0, - "24": 1597485568.0, - "25": 1597485568.0, - "26": 1597485568.0, - "27": 1597485568.0, - "28": 1597485568.0, - "29": 1597485568.0, - "30": 1597485568.0, - "31": 1597485568.0, - "32": 1597485568.0, - "33": 1597485568.0, - "34": 1597485568.0, - "35": 1597485568.0, - "36": 1597485568.0, - "37": 1597485568.0, - "38": 1597485568.0, - "39": 1597485568.0, - "40": 1597485568.0, - "41": 1597485568.0, - "42": 1597485568.0, - "43": 1597485568.0, - "44": 1597485568.0, - "45": 1597485568.0, - "46": 1597485568.0, - "47": 1597485568.0, - "48": 1597485568.0, - "49": 1597485568.0, - "50": 1597485568.0, - "51": 1597485568.0, - "52": 1597485568.0, - "53": 1597485568.0, - "54": 1597485568.0, - "55": 1597485568.0, - "56": 1597485568.0, - "57": 1597485568.0, - "58": 1597485568.0, - "59": 1597485568.0, - "60": 1597485568.0, - "61": 1597485568.0, - "62": 1597485568.0, - "63": 1597485568.0, - "64": 1597485568.0, - "65": 1597485568.0, - "66": 1597485568.0, - "67": 1597485568.0, - "68": 1597485568.0, - "69": 1597485568.0, - "70": 1597485568.0, - "71": 1597485568.0, - "72": 1597485568.0, - "73": 1597485568.0, - "74": 1597485568.0, - "75": 1597485568.0, - "76": 1597485568.0, - "77": 1597485568.0, - "78": 1597485568.0, - "79": 1597485568.0, - "80": 1597485568.0, - "81": 1597485568.0, - "82": 1597485568.0, - "83": 1597485568.0, - "84": 1597485568.0, - "85": 1597485568.0, - "86": 1597485568.0, - "87": 1597485568.0, - "88": 1597485568.0, - "89": 1597485568.0, - "90": 1597485568.0, - "91": 1597485568.0, - "92": 1597485568.0, - "93": 1597485568.0, - "94": 1597485568.0, - "95": 1597485568.0, - "96": 1597485568.0, - "97": 1597485568.0, - "98": 1597485568.0, - "99": 1597485568.0, - "100": 1597485568.0 + "1": 1464319488.0, + "2": 1464320000.0, + "3": 1464320000.0, + "4": 1464320000.0, + "5": 1464320000.0, + "6": 1464320000.0, + "7": 1464320000.0, + "8": 1464320000.0, + "9": 1464320000.0, + "10": 1464320000.0, + "11": 1464320000.0, + "12": 1464320000.0, + "13": 1464320000.0, + "14": 1464320000.0, + "15": 1464320000.0, + "16": 1464320000.0, + "17": 1597092352.0, + "18": 1597092352.0, + "19": 1597092352.0, + "20": 1597092352.0, + "21": 1597092352.0, + "22": 1597092352.0, + "23": 1597092352.0, + "24": 1597092352.0, + "25": 1597092352.0, + "26": 1597092352.0, + "27": 1597092352.0, + "28": 1597092352.0, + "29": 1597092352.0, + "30": 1597092352.0, + "31": 1597092352.0, + "32": 1597092352.0, + "33": 1597092352.0, + "34": 1597092352.0, + "35": 1597092352.0, + "36": 1597092352.0, + "37": 1597092352.0, + "38": 1597092352.0, + "39": 1597092352.0, + "40": 1597092352.0, + "41": 1597092352.0, + "42": 1597092352.0, + "43": 1597092352.0, + "44": 1597092352.0, + "45": 1597092352.0, + "46": 1597092352.0, + "47": 1597092352.0, + "48": 1597092352.0, + "49": 1597092352.0, + "50": 1597092352.0, + "51": 1597092352.0, + "52": 1597092352.0, + "53": 1597092352.0, + "54": 1597092352.0, + "55": 1597092352.0, + "56": 1597092352.0, + "57": 1597092352.0, + "58": 1597092352.0, + "59": 1597092352.0, + "60": 1597092352.0, + "61": 1597092352.0, + "62": 1597092352.0, + "63": 1597092352.0, + "64": 1597092352.0, + "65": 1597092352.0, + "66": 1597092352.0, + "67": 1597092352.0, + "68": 1597092352.0, + "69": 1597092352.0, + "70": 1597092352.0, + "71": 1597092352.0, + "72": 1597092352.0, + "73": 1597092352.0, + "74": 1597092352.0, + "75": 1597092352.0, + "76": 1597092352.0, + "77": 1597092352.0, + "78": 1597092352.0, + "79": 1597092352.0, + "80": 1597092352.0, + "81": 1597092352.0, + "82": 1597092352.0, + "83": 1597092352.0, + "84": 1597092352.0, + "85": 1597092352.0, + "86": 1597092352.0, + "87": 1597092352.0, + "88": 1597092352.0, + "89": 1597092352.0, + "90": 1597092352.0, + "91": 1597092352.0, + "92": 1597092352.0, + "93": 1597092352.0, + "94": 1597092352.0, + "95": 1597092352.0, + "96": 1597092352.0, + "97": 1597092352.0, + "98": 1597092352.0, + "99": 1597092352.0, + "100": 1597092352.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.41128, - "3": 0.32723, - "4": 0.31051, - "5": 0.30949, - "6": 0.31179, - "7": 0.30989, - "8": 0.30991, - "9": 0.31033, - "10": 0.31023, - "11": 0.31525, - "12": 0.31868, - "13": 0.31199, - "14": 0.31155, - "15": 0.31266, - "16": 0.44853, - "17": 0.33588, - "18": 0.33074, - "19": 0.32991, - "20": 0.32915, - "21": 0.32894, - "22": 0.32936, - "23": 0.32867, - "24": 0.33005, - "25": 0.33353, - "26": 0.32991, - "27": 0.32657, - "28": 0.32936, - "29": 0.32904, - "30": 0.33004, - "31": 0.33224, - "32": 0.33099, - "33": 0.33176, - "34": 0.33261, - "35": 0.32912, - "36": 0.32941, - "37": 0.33241, - "38": 0.33129, - "39": 0.33114, - "40": 0.33312, - "41": 0.33131, - "42": 0.32999, - "43": 0.33215, - "44": 0.33311, - "45": 0.33235, - "46": 0.33253, - "47": 0.33286, - "48": 0.33252, - "49": 0.33391, - "50": 0.33225, - "51": 0.43781, - "52": 0.33165, - "53": 0.33102, - "54": 0.33239, - "55": 0.334, - "56": 0.33117, - "57": 0.32917, - "58": 0.33279, - "59": 0.33208, - "60": 0.33419, - "61": 0.33315, - "62": 0.33491, - "63": 0.33168, - "64": 0.33089, - "65": 0.33337, - "66": 0.33225, - "67": 0.33277, - "68": 0.33177, - "69": 0.33314, - "70": 0.33167, - "71": 0.332, - "72": 0.33353, - "73": 0.33202, - "74": 0.33204, - "75": 0.33419, - "76": 0.33075, - "77": 0.33163, - "78": 0.33217, - "79": 0.33264, - "80": 0.33112, - "81": 0.33377, - "82": 0.33288, - "83": 0.33162, - "84": 0.33228, - "85": 0.33438, - "86": 0.33394, - "87": 0.33487, - "88": 0.33304, - "89": 0.3322, - "90": 0.3333, - "91": 0.33264, - "92": 0.33331, - "93": 0.33375, - "94": 0.33306, - "95": 0.33408, - "96": 0.33344, - "97": 0.33266, - "98": 0.33176, - "99": 0.36412, - "100": 0.41619 + "2": 4.57766, + "3": 0.31828, + "4": 0.26985, + "5": 0.26865, + "6": 0.26983, + "7": 0.27088, + "8": 0.2711, + "9": 0.27178, + "10": 0.27248, + "11": 0.2719, + "12": 0.27178, + "13": 0.27116, + "14": 0.26942, + "15": 0.27027, + "16": 0.43235, + "17": 0.28713, + "18": 0.28073, + "19": 0.28246, + "20": 0.28153, + "21": 0.28258, + "22": 0.28234, + "23": 0.28032, + "24": 0.27922, + "25": 0.28158, + "26": 0.28326, + "27": 0.28204, + "28": 0.2823, + "29": 0.28356, + "30": 0.28398, + "31": 0.28274, + "32": 0.28269, + "33": 0.28317, + "34": 0.28182, + "35": 0.2842, + "36": 0.28502, + "37": 0.28223, + "38": 0.28219, + "39": 0.28204, + "40": 0.28093, + "41": 0.28238, + "42": 0.28406, + "43": 0.28202, + "44": 0.28305, + "45": 0.28124, + "46": 0.28422, + "47": 0.28227, + "48": 0.28152, + "49": 0.28248, + "50": 0.28165, + "51": 0.42668, + "52": 0.28193, + "53": 0.27786, + "54": 0.27986, + "55": 0.27863, + "56": 0.2796, + "57": 0.28684, + "58": 0.28475, + "59": 0.28004, + "60": 0.28103, + "61": 0.27928, + "62": 0.27835, + "63": 0.27827, + "64": 0.27913, + "65": 0.27858, + "66": 0.27887, + "67": 0.28015, + "68": 0.27814, + "69": 0.27874, + "70": 0.28069, + "71": 0.27764, + "72": 0.2799, + "73": 0.27846, + "74": 0.27918, + "75": 0.27771, + "76": 0.27963, + "77": 0.27787, + "78": 0.28144, + "79": 0.27797, + "80": 0.27927, + "81": 0.28031, + "82": 0.28022, + "83": 0.28159, + "84": 0.28008, + "85": 0.27961, + "86": 0.28185, + "87": 0.28105, + "88": 0.28254, + "89": 0.28103, + "90": 0.27719, + "91": 0.27925, + "92": 0.27823, + "93": 0.27758, + "94": 0.2812, + "95": 0.27785, + "96": 0.27959, + "97": 0.28024, + "98": 0.28022, + "99": 0.27861, + "100": 0.27796 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2159.0, - "17": 2560.0, - "18": 2249.0, - "19": 2176.0, - "20": 2256.0, - "21": 2385.0, - "22": 2063.0, - "23": 1983.0, - "24": 2238.0, - "25": 1971.0, - "26": 2208.0, - "27": 2282.0, - "28": 2223.0, - "29": 2238.0, - "30": 1973.0, - "31": 2425.0, - "32": 2295.0, - "33": 2048.0, - "34": 2303.0, - "35": 2268.0, - "36": 2224.0, - "37": 2273.0, - "38": 2550.0, - "39": 2326.0, - "40": 2239.0, - "41": 2323.0, - "42": 2008.0, - "43": 2580.0, - "44": 2073.0, - "45": 2678.0, - "46": 2212.0, - "47": 2351.0, - "48": 2373.0, - "49": 2572.0, - "50": 2384.0, - "51": 2145.0, - "52": 2552.0, - "53": 2588.0, - "54": 2412.0, - "55": 2353.0, - "56": 2664.0, - "57": 2263.0, - "58": 3299.0, - "59": 2567.0, - "60": 3005.0, - "61": 2438.0, - "62": 3160.0, - "63": 3270.0, - "64": 2785.0, - "65": 2938.0, - "66": 3077.0, - "67": 3347.0, - "68": 3337.0, - "69": 3026.0, - "70": 3635.0, - "71": 2916.0, - "72": 2982.0, - "73": 3405.0, - "74": 3143.0, - "75": 3874.0, - "76": 3635.0, - "77": 3466.0, - "78": 3688.0, - "79": 5543.0, - "80": 3722.0, - "81": 4065.0, - "82": 2887.0, - "83": 3352.0, - "84": 3929.0, - "85": 4056.0, - "86": 3472.0, - "87": 2663.0, - "88": 4257.0, - "89": 4063.0, - "90": 5060.0, - "91": 2917.0, - "92": 3977.0, - "93": 3920.0, - "94": 3284.0, - "95": 3709.0, - "96": 4570.0, - "97": 4467.0, - "98": 4380.0, - "99": 3448.0, - "100": 3807.0 + "16": 2090.0, + "17": 2642.0, + "18": 2323.0, + "19": 2072.0, + "20": 2358.0, + "21": 2376.0, + "22": 2082.0, + "23": 2046.0, + "24": 2235.0, + "25": 1922.0, + "26": 2167.0, + "27": 2220.0, + "28": 2332.0, + "29": 2472.0, + "30": 1988.0, + "31": 2497.0, + "32": 2266.0, + "33": 2105.0, + "34": 2329.0, + "35": 2275.0, + "36": 2211.0, + "37": 2270.0, + "38": 2454.0, + "39": 2403.0, + "40": 2248.0, + "41": 2288.0, + "42": 2035.0, + "43": 2514.0, + "44": 2137.0, + "45": 2595.0, + "46": 2270.0, + "47": 2250.0, + "48": 2392.0, + "49": 2483.0, + "50": 2450.0, + "51": 2299.0, + "52": 2579.0, + "53": 2494.0, + "54": 2311.0, + "55": 2398.0, + "56": 2637.0, + "57": 2314.0, + "58": 3264.0, + "59": 2662.0, + "60": 2967.0, + "61": 2462.0, + "62": 3097.0, + "63": 3247.0, + "64": 2926.0, + "65": 2977.0, + "66": 3103.0, + "67": 3364.0, + "68": 3387.0, + "69": 2975.0, + "70": 3809.0, + "71": 2902.0, + "72": 2994.0, + "73": 3386.0, + "74": 3231.0, + "75": 3811.0, + "76": 3648.0, + "77": 3489.0, + "78": 3550.0, + "79": 5447.0, + "80": 3663.0, + "81": 4094.0, + "82": 2990.0, + "83": 3287.0, + "84": 3784.0, + "85": 4049.0, + "86": 3425.0, + "87": 2647.0, + "88": 4355.0, + "89": 4137.0, + "90": 5098.0, + "91": 2799.0, + "92": 3843.0, + "93": 3980.0, + "94": 3246.0, + "95": 3807.0, + "96": 4497.0, + "97": 4540.0, + "98": 4336.0, + "99": 3437.0, + "100": 3918.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_h100.json index 0b984afdd9a..a4a4686c9dd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_h100.json @@ -4,104 +4,104 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89757, - "2": 10.89076, - "3": 10.88414, - "4": 10.89075, + "1": 10.89758, + "2": 10.89074, + "3": 10.88416, + "4": 10.89076, "5": 10.88375, - "6": 10.88793, - "7": 10.88856, - "8": 10.88592, - "9": 10.88899, - "10": 10.8852, - "11": 10.89078, + "6": 10.88788, + "7": 10.88855, + "8": 10.88593, + "9": 10.88903, + "10": 10.88521, + "11": 10.8908, "12": 10.89125, - "13": 10.88783, - "14": 10.88523, + "13": 10.88782, + "14": 10.88524, "15": 10.88844, - "16": 10.88579, - "17": 10.88587, - "18": 10.88812, + "16": 10.88577, + "17": 10.88588, + "18": 10.88813, "19": 10.89138, - "20": 10.88198, - "21": 10.88925, - "22": 10.875, + "20": 10.88194, + "21": 10.88923, + "22": 10.87496, "23": 10.86547, - "24": 10.86171, - "25": 10.86263, - "26": 10.83462, - "27": 10.83634, - "28": 10.82662, - "29": 10.81094, + "24": 10.86172, + "25": 10.86267, + "26": 10.8346, + "27": 10.83637, + "28": 10.8266, + "29": 10.81092, "30": 10.78664, - "31": 10.78372, - "32": 10.77579, - "33": 10.7342, + "31": 10.78369, + "32": 10.7758, + "33": 10.73418, "34": 10.71119, - "35": 10.70532, - "36": 10.69049, - "37": 10.65866, - "38": 10.64974, - "39": 10.62381, - "40": 10.6069, + "35": 10.70535, + "36": 10.6905, + "37": 10.65867, + "38": 10.64971, + "39": 10.62382, + "40": 10.60691, "41": 10.57833, - "42": 10.5603, - "43": 10.54222, + "42": 10.56029, + "43": 10.54219, "44": 10.5188, - "45": 10.5053, - "46": 10.48038, - "47": 10.46032, - "48": 10.41509, - "49": 10.41634, - "50": 10.40081, - "51": 10.39365, - "52": 10.34945, + "45": 10.50527, + "46": 10.4804, + "47": 10.46031, + "48": 10.41508, + "49": 10.41632, + "50": 10.4008, + "51": 10.39364, + "52": 10.34944, "53": 10.342, - "54": 10.31227, - "55": 10.29325, + "54": 10.31225, + "55": 10.29323, "56": 10.29329, - "57": 10.2813, - "58": 10.28129, - "59": 10.21997, + "57": 10.28126, + "58": 10.28128, + "59": 10.21996, "60": 10.23018, - "61": 10.18393, + "61": 10.18391, "62": 10.15993, "63": 10.20602, - "64": 10.16203, + "64": 10.16204, "65": 10.12249, "66": 10.15236, - "67": 10.12965, - "68": 10.09494, + "67": 10.12963, + "68": 10.09493, "69": 10.10681, - "70": 10.08342, - "71": 10.09959, - "72": 10.08977, - "73": 10.07514, + "70": 10.0834, + "71": 10.09957, + "72": 10.08974, + "73": 10.07511, "74": 10.06651, - "75": 10.03809, - "76": 10.05237, - "77": 10.05049, - "78": 10.00922, - "79": 10.01367, - "80": 10.03041, - "81": 10.05065, - "82": 9.98611, - "83": 9.96212, - "84": 9.89756, + "75": 10.03807, + "76": 10.05239, + "77": 10.05048, + "78": 10.00921, + "79": 10.01364, + "80": 10.03039, + "81": 10.05063, + "82": 9.98609, + "83": 9.96211, + "84": 9.89755, "85": 9.89373, "86": 9.9881, "87": 9.99417, - "88": 9.97617, - "89": 9.92167, - "90": 9.91508, - "91": 9.94041, - "92": 9.91744, - "93": 9.85398, - "94": 9.92293, - "95": 9.90966, + "88": 9.97615, + "89": 9.92166, + "90": 9.91507, + "91": 9.94039, + "92": 9.91741, + "93": 9.85395, + "94": 9.92292, + "95": 9.90964, "96": 9.88828, - "97": 9.84573, - "98": 9.86176, + "97": 9.84571, + "98": 9.86174, "99": 9.91044, "100": 9.80688 } @@ -126,91 +126,91 @@ "13": 284527616.0, "14": 284527616.0, "15": 284527616.0, - "16": 416513536.0, - "17": 416513536.0, - "18": 416513536.0, - "19": 416513536.0, - "20": 416513536.0, - "21": 416513536.0, - "22": 416513536.0, - "23": 416513536.0, - "24": 416513536.0, - "25": 416513536.0, - "26": 416513536.0, - "27": 416513536.0, - "28": 416513536.0, - "29": 416513536.0, - "30": 416513536.0, - "31": 416513536.0, - "32": 416513536.0, - "33": 416513536.0, - "34": 416513536.0, - "35": 416513536.0, - "36": 416513536.0, - "37": 416513536.0, - "38": 416513536.0, - "39": 416513536.0, - "40": 416513536.0, - "41": 416513536.0, - "42": 416513536.0, - "43": 416513536.0, - "44": 416513536.0, - "45": 416513536.0, - "46": 416513536.0, - "47": 416513536.0, - "48": 416513536.0, - "49": 416513536.0, - "50": 416513536.0, - "51": 416513536.0, - "52": 416513536.0, - "53": 416513536.0, - "54": 416513536.0, - "55": 416513536.0, - "56": 416513536.0, - "57": 416513536.0, - "58": 416513536.0, - "59": 416513536.0, - "60": 416513536.0, - "61": 416513536.0, - "62": 416513536.0, - "63": 416513536.0, - "64": 416513536.0, - "65": 416513536.0, - "66": 416513536.0, - "67": 416513536.0, - "68": 416513536.0, - "69": 416513536.0, - "70": 416513536.0, - "71": 416513536.0, - "72": 416513536.0, - "73": 416513536.0, - "74": 416513536.0, - "75": 416513536.0, - "76": 416513536.0, - "77": 416513536.0, - "78": 416513536.0, - "79": 416513536.0, - "80": 416513536.0, - "81": 416513536.0, - "82": 416513536.0, - "83": 416513536.0, - "84": 416513536.0, - "85": 416513536.0, - "86": 416513536.0, - "87": 416513536.0, - "88": 416513536.0, - "89": 416513536.0, - "90": 416513536.0, - "91": 416513536.0, - "92": 416513536.0, - "93": 416513536.0, - "94": 416513536.0, - "95": 416513536.0, - "96": 416513536.0, - "97": 416513536.0, - "98": 416513536.0, - "99": 416513536.0, - "100": 416513536.0 + "16": 417037824.0, + "17": 417037824.0, + "18": 417037824.0, + "19": 417037824.0, + "20": 417037824.0, + "21": 417037824.0, + "22": 417037824.0, + "23": 417037824.0, + "24": 417037824.0, + "25": 417037824.0, + "26": 417037824.0, + "27": 417037824.0, + "28": 417037824.0, + "29": 417037824.0, + "30": 417037824.0, + "31": 417037824.0, + "32": 417037824.0, + "33": 417037824.0, + "34": 417037824.0, + "35": 417037824.0, + "36": 417037824.0, + "37": 417037824.0, + "38": 417037824.0, + "39": 417037824.0, + "40": 417037824.0, + "41": 417037824.0, + "42": 417037824.0, + "43": 417037824.0, + "44": 417037824.0, + "45": 417037824.0, + "46": 417037824.0, + "47": 417037824.0, + "48": 417037824.0, + "49": 417037824.0, + "50": 417037824.0, + "51": 417037824.0, + "52": 417037824.0, + "53": 417037824.0, + "54": 417037824.0, + "55": 417037824.0, + "56": 417037824.0, + "57": 417037824.0, + "58": 417037824.0, + "59": 417037824.0, + "60": 417037824.0, + "61": 417037824.0, + "62": 417037824.0, + "63": 417037824.0, + "64": 417037824.0, + "65": 417037824.0, + "66": 417037824.0, + "67": 417037824.0, + "68": 417037824.0, + "69": 417037824.0, + "70": 417037824.0, + "71": 417037824.0, + "72": 417037824.0, + "73": 417037824.0, + "74": 417037824.0, + "75": 417037824.0, + "76": 417037824.0, + "77": 417037824.0, + "78": 417037824.0, + "79": 417037824.0, + "80": 417037824.0, + "81": 417037824.0, + "82": 417037824.0, + "83": 417037824.0, + "84": 417037824.0, + "85": 417037824.0, + "86": 417037824.0, + "87": 417037824.0, + "88": 417037824.0, + "89": 417037824.0, + "90": 417037824.0, + "91": 417037824.0, + "92": 417037824.0, + "93": 417037824.0, + "94": 417037824.0, + "95": 417037824.0, + "96": 417037824.0, + "97": 417037824.0, + "98": 417037824.0, + "99": 417037824.0, + "100": 417037824.0 } }, "mem-max-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1465368064.0, - "2": 1465368576.0, - "3": 1465368576.0, - "4": 1465368576.0, - "5": 1465368576.0, - "6": 1465368576.0, - "7": 1465368576.0, - "8": 1465368576.0, - "9": 1465368576.0, - "10": 1465368576.0, - "11": 1465368576.0, - "12": 1465368576.0, - "13": 1465368576.0, - "14": 1465368576.0, - "15": 1465368576.0, - "16": 1465368576.0, - "17": 1597485568.0, - "18": 1597485568.0, - "19": 1597485568.0, - "20": 1597485568.0, - "21": 1597485568.0, - "22": 1597485568.0, - "23": 1597485568.0, - "24": 1597485568.0, - "25": 1597485568.0, - "26": 1597485568.0, - "27": 1597485568.0, - "28": 1597485568.0, - "29": 1597485568.0, - "30": 1597485568.0, - "31": 1597485568.0, - "32": 1597485568.0, - "33": 1597485568.0, - "34": 1597485568.0, - "35": 1597485568.0, - "36": 1597485568.0, - "37": 1597485568.0, - "38": 1597485568.0, - "39": 1597485568.0, - "40": 1597485568.0, - "41": 1597485568.0, - "42": 1597485568.0, - "43": 1597485568.0, - "44": 1597485568.0, - "45": 1597485568.0, - "46": 1597485568.0, - "47": 1597485568.0, - "48": 1597485568.0, - "49": 1597485568.0, - "50": 1597485568.0, - "51": 1597485568.0, - "52": 1597485568.0, - "53": 1597485568.0, - "54": 1597485568.0, - "55": 1597485568.0, - "56": 1597485568.0, - "57": 1597485568.0, - "58": 1597485568.0, - "59": 1597485568.0, - "60": 1597485568.0, - "61": 1597485568.0, - "62": 1597485568.0, - "63": 1597485568.0, - "64": 1597485568.0, - "65": 1597485568.0, - "66": 1597485568.0, - "67": 1597485568.0, - "68": 1597485568.0, - "69": 1597485568.0, - "70": 1597485568.0, - "71": 1597485568.0, - "72": 1597485568.0, - "73": 1597485568.0, - "74": 1597485568.0, - "75": 1597485568.0, - "76": 1597485568.0, - "77": 1597485568.0, - "78": 1597485568.0, - "79": 1597485568.0, - "80": 1597485568.0, - "81": 1597485568.0, - "82": 1597485568.0, - "83": 1597485568.0, - "84": 1597485568.0, - "85": 1597485568.0, - "86": 1597485568.0, - "87": 1597485568.0, - "88": 1597485568.0, - "89": 1597485568.0, - "90": 1597485568.0, - "91": 1597485568.0, - "92": 1597485568.0, - "93": 1597485568.0, - "94": 1597485568.0, - "95": 1597485568.0, - "96": 1597485568.0, - "97": 1597485568.0, - "98": 1597485568.0, - "99": 1597485568.0, - "100": 1597485568.0 + "1": 1464319488.0, + "2": 1464320000.0, + "3": 1464320000.0, + "4": 1464320000.0, + "5": 1464320000.0, + "6": 1464320000.0, + "7": 1464320000.0, + "8": 1464320000.0, + "9": 1464320000.0, + "10": 1464320000.0, + "11": 1464320000.0, + "12": 1464320000.0, + "13": 1464320000.0, + "14": 1464320000.0, + "15": 1464320000.0, + "16": 1464320000.0, + "17": 1597616640.0, + "18": 1597616640.0, + "19": 1597616640.0, + "20": 1597616640.0, + "21": 1597616640.0, + "22": 1597616640.0, + "23": 1597616640.0, + "24": 1597616640.0, + "25": 1597616640.0, + "26": 1597616640.0, + "27": 1597616640.0, + "28": 1597616640.0, + "29": 1597616640.0, + "30": 1597616640.0, + "31": 1597616640.0, + "32": 1597616640.0, + "33": 1597616640.0, + "34": 1597616640.0, + "35": 1597616640.0, + "36": 1597616640.0, + "37": 1597616640.0, + "38": 1597616640.0, + "39": 1597616640.0, + "40": 1597616640.0, + "41": 1597616640.0, + "42": 1597616640.0, + "43": 1597616640.0, + "44": 1597616640.0, + "45": 1597616640.0, + "46": 1597616640.0, + "47": 1597616640.0, + "48": 1597616640.0, + "49": 1597616640.0, + "50": 1597616640.0, + "51": 1598662656.0, + "52": 1598662656.0, + "53": 1598662656.0, + "54": 1598662656.0, + "55": 1598662656.0, + "56": 1598662656.0, + "57": 1598662656.0, + "58": 1598662656.0, + "59": 1598662656.0, + "60": 1598662656.0, + "61": 1598662656.0, + "62": 1598662656.0, + "63": 1598662656.0, + "64": 1598662656.0, + "65": 1598662656.0, + "66": 1598662656.0, + "67": 1598662656.0, + "68": 1598662656.0, + "69": 1598662656.0, + "70": 1598662656.0, + "71": 1598662656.0, + "72": 1598662656.0, + "73": 1598662656.0, + "74": 1598662656.0, + "75": 1598662656.0, + "76": 1598662656.0, + "77": 1598662656.0, + "78": 1598662656.0, + "79": 1598662656.0, + "80": 1598662656.0, + "81": 1598662656.0, + "82": 1598662656.0, + "83": 1598662656.0, + "84": 1598662656.0, + "85": 1598662656.0, + "86": 1598662656.0, + "87": 1598662656.0, + "88": 1598662656.0, + "89": 1598662656.0, + "90": 1598662656.0, + "91": 1598662656.0, + "92": 1598662656.0, + "93": 1598662656.0, + "94": 1598662656.0, + "95": 1598662656.0, + "96": 1598662656.0, + "97": 1598662656.0, + "98": 1598662656.0, + "99": 1598662656.0, + "100": 1598662656.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.94014, - "3": 0.2065, - "4": 0.19124, - "5": 0.18873, - "6": 0.18906, - "7": 0.18962, - "8": 0.18957, - "9": 0.18877, - "10": 0.19006, - "11": 0.189, - "12": 0.18911, - "13": 0.19198, - "14": 0.18868, - "15": 0.18946, - "16": 0.27424, - "17": 0.20461, - "18": 0.20208, - "19": 0.20473, - "20": 0.20221, - "21": 0.203, - "22": 0.20165, - "23": 0.20177, - "24": 0.20159, - "25": 0.20148, - "26": 0.20463, - "27": 0.20292, - "28": 0.20282, - "29": 0.20248, - "30": 0.20225, - "31": 0.2021, - "32": 0.20378, - "33": 0.20291, - "34": 0.2021, - "35": 0.20234, - "36": 0.20371, - "37": 0.20271, - "38": 0.20223, - "39": 0.20287, - "40": 0.20315, - "41": 0.20487, - "42": 0.20176, - "43": 0.20168, - "44": 0.20297, - "45": 0.20568, - "46": 0.20219, - "47": 0.20255, - "48": 0.20166, - "49": 0.19359, - "50": 0.19056, - "51": 0.23555, - "52": 0.19583, - "53": 0.19035, - "54": 0.19404, - "55": 0.19527, - "56": 0.19112, - "57": 0.19198, - "58": 0.19069, - "59": 0.19078, - "60": 0.19147, - "61": 0.19114, - "62": 0.19128, - "63": 0.19234, - "64": 0.19236, - "65": 0.19052, - "66": 0.19317, - "67": 0.19302, - "68": 0.19128, - "69": 0.19143, - "70": 0.19327, - "71": 0.19339, - "72": 0.19097, - "73": 0.19108, - "74": 0.1913, - "75": 0.19083, - "76": 0.19149, - "77": 0.19062, - "78": 0.19098, - "79": 0.19059, - "80": 0.19086, - "81": 0.19164, - "82": 0.19068, - "83": 0.19103, - "84": 0.18997, - "85": 0.1907, - "86": 0.19036, - "87": 0.1918, - "88": 0.19021, - "89": 0.19031, - "90": 0.19025, - "91": 0.19153, - "92": 0.19158, - "93": 0.19093, - "94": 0.19051, - "95": 0.19185, - "96": 0.19178, - "97": 0.19145, - "98": 0.19081, - "99": 0.19082, - "100": 0.19126 + "2": 3.51639, + "3": 0.22612, + "4": 0.19945, + "5": 0.19782, + "6": 0.19727, + "7": 0.20018, + "8": 0.19699, + "9": 0.19806, + "10": 0.19616, + "11": 0.19592, + "12": 0.19725, + "13": 0.19705, + "14": 0.1964, + "15": 0.19658, + "16": 0.28248, + "17": 0.20917, + "18": 0.20569, + "19": 0.20525, + "20": 0.20465, + "21": 0.20739, + "22": 0.2064, + "23": 0.2211, + "24": 0.21772, + "25": 0.2054, + "26": 0.22221, + "27": 0.21585, + "28": 0.21162, + "29": 0.20334, + "30": 0.20237, + "31": 0.2108, + "32": 0.24207, + "33": 0.20934, + "34": 0.20756, + "35": 0.20714, + "36": 0.20597, + "37": 0.20277, + "38": 0.2074, + "39": 0.20378, + "40": 0.20712, + "41": 0.206, + "42": 0.20492, + "43": 0.205, + "44": 0.2048, + "45": 0.20519, + "46": 0.20553, + "47": 0.20514, + "48": 0.20456, + "49": 0.20568, + "50": 0.20519, + "51": 0.25236, + "52": 0.21268, + "53": 0.21571, + "54": 0.21343, + "55": 0.21248, + "56": 0.21205, + "57": 0.2147, + "58": 0.21319, + "59": 0.21375, + "60": 0.21389, + "61": 0.21312, + "62": 0.20932, + "63": 0.20875, + "64": 0.20852, + "65": 0.20778, + "66": 0.20811, + "67": 0.20687, + "68": 0.20731, + "69": 0.20773, + "70": 0.20741, + "71": 0.20674, + "72": 0.20683, + "73": 0.20691, + "74": 0.20604, + "75": 0.20561, + "76": 0.20738, + "77": 0.20644, + "78": 0.20714, + "79": 0.20789, + "80": 0.20832, + "81": 0.20787, + "82": 0.20873, + "83": 0.20692, + "84": 0.20689, + "85": 0.20711, + "86": 0.20649, + "87": 0.2062, + "88": 0.2126, + "89": 0.21318, + "90": 0.21513, + "91": 0.21662, + "92": 0.21351, + "93": 0.214, + "94": 0.21619, + "95": 0.21355, + "96": 0.21423, + "97": 0.21528, + "98": 0.21612, + "99": 0.21291, + "100": 0.21346 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2012.0, - "17": 2525.0, - "18": 2270.0, - "19": 1958.0, - "20": 2327.0, - "21": 2331.0, - "22": 2160.0, - "23": 2112.0, - "24": 2098.0, - "25": 1958.0, - "26": 2110.0, - "27": 2245.0, - "28": 2284.0, - "29": 2351.0, - "30": 1913.0, - "31": 2467.0, - "32": 2294.0, - "33": 2082.0, - "34": 2307.0, - "35": 2318.0, - "36": 2250.0, - "37": 2177.0, - "38": 2457.0, - "39": 2281.0, - "40": 2255.0, - "41": 2199.0, - "42": 2096.0, - "43": 2430.0, - "44": 2123.0, - "45": 2627.0, - "46": 2254.0, - "47": 2300.0, - "48": 2352.0, - "49": 2469.0, - "50": 2494.0, - "51": 2240.0, - "52": 2577.0, - "53": 2590.0, - "54": 2294.0, - "55": 2263.0, - "56": 2662.0, - "57": 2276.0, - "58": 3221.0, - "59": 2539.0, - "60": 3055.0, - "61": 2438.0, - "62": 2965.0, - "63": 3320.0, - "64": 2754.0, + "16": 1982.0, + "17": 2464.0, + "18": 2254.0, + "19": 2029.0, + "20": 2317.0, + "21": 2275.0, + "22": 2138.0, + "23": 2073.0, + "24": 2157.0, + "25": 1998.0, + "26": 2137.0, + "27": 2237.0, + "28": 2354.0, + "29": 2392.0, + "30": 1955.0, + "31": 2503.0, + "32": 2238.0, + "33": 2130.0, + "34": 2354.0, + "35": 2256.0, + "36": 2217.0, + "37": 2234.0, + "38": 2462.0, + "39": 2264.0, + "40": 2217.0, + "41": 2294.0, + "42": 2136.0, + "43": 2485.0, + "44": 2174.0, + "45": 2605.0, + "46": 2196.0, + "47": 2351.0, + "48": 2332.0, + "49": 2449.0, + "50": 2391.0, + "51": 2232.0, + "52": 2589.0, + "53": 2541.0, + "54": 2326.0, + "55": 2231.0, + "56": 2615.0, + "57": 2232.0, + "58": 3310.0, + "59": 2553.0, + "60": 2945.0, + "61": 2422.0, + "62": 2943.0, + "63": 3285.0, + "64": 2761.0, "65": 2866.0, - "66": 3104.0, - "67": 3322.0, - "68": 3009.0, - "69": 3106.0, - "70": 3656.0, - "71": 2897.0, - "72": 2893.0, - "73": 3254.0, - "74": 3116.0, - "75": 3898.0, - "76": 3522.0, - "77": 3521.0, - "78": 3507.0, - "79": 5478.0, - "80": 3514.0, - "81": 4336.0, - "82": 3007.0, - "83": 3300.0, - "84": 3941.0, - "85": 3973.0, - "86": 3457.0, - "87": 2705.0, - "88": 4224.0, - "89": 4308.0, + "66": 3079.0, + "67": 3245.0, + "68": 3159.0, + "69": 3124.0, + "70": 3627.0, + "71": 2888.0, + "72": 3051.0, + "73": 3295.0, + "74": 3149.0, + "75": 3945.0, + "76": 3549.0, + "77": 3559.0, + "78": 3516.0, + "79": 5398.0, + "80": 3557.0, + "81": 4303.0, + "82": 2965.0, + "83": 3422.0, + "84": 3906.0, + "85": 4042.0, + "86": 3582.0, + "87": 2704.0, + "88": 4189.0, + "89": 4228.0, "90": 5060.0, - "91": 2666.0, - "92": 3783.0, - "93": 3904.0, - "94": 3305.0, - "95": 3553.0, - "96": 4233.0, - "97": 4488.0, - "98": 4417.0, - "99": 3247.0, - "100": 3493.0 + "91": 2720.0, + "92": 3889.0, + "93": 3943.0, + "94": 3315.0, + "95": 3539.0, + "96": 4499.0, + "97": 4487.0, + "98": 4457.0, + "99": 3230.0, + "100": 3424.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json index 44874fe165c..d3d4abf8d06 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89629, - "2": 10.88412, + "1": 10.89631, + "2": 10.88413, "3": 10.88476, "4": 10.88014, - "5": 10.8825, - "6": 10.87881, - "7": 10.8837, - "8": 10.88065, - "9": 10.88602, + "5": 10.88249, + "6": 10.8788, + "7": 10.88368, + "8": 10.88067, + "9": 10.88608, "10": 10.88235, - "11": 10.88644, - "12": 10.88403, - "13": 10.88226, - "14": 10.88319, - "15": 10.8795, - "16": 10.88084, + "11": 10.88646, + "12": 10.884, + "13": 10.88224, + "14": 10.88321, + "15": 10.87947, + "16": 10.8808, "17": 10.89482, "18": 10.87831, - "19": 10.89758, - "20": 10.87963, - "21": 10.88892, - "22": 10.88443, + "19": 10.89755, + "20": 10.87959, + "21": 10.88893, + "22": 10.88442, "23": 10.87227, - "24": 10.85702, + "24": 10.85705, "25": 10.86099, - "26": 10.83645, - "27": 10.83445, - "28": 10.83039, - "29": 10.8082, - "30": 10.78668, - "31": 10.77883, + "26": 10.83647, + "27": 10.83442, + "28": 10.83038, + "29": 10.80813, + "30": 10.78669, + "31": 10.7788, "32": 10.77361, "33": 10.74029, - "34": 10.72355, + "34": 10.72353, "35": 10.71304, - "36": 10.68845, - "37": 10.66097, + "36": 10.6884, + "37": 10.66094, "38": 10.65601, - "39": 10.63174, - "40": 10.61698, - "41": 10.58755, - "42": 10.5645, - "43": 10.54922, + "39": 10.63172, + "40": 10.61696, + "41": 10.58756, + "42": 10.56449, + "43": 10.54924, "44": 10.52112, - "45": 10.51181, - "46": 10.48416, - "47": 10.4652, + "45": 10.5118, + "46": 10.48418, + "47": 10.46518, "48": 10.42505, - "49": 10.41707, - "50": 10.40407, - "51": 10.39841, + "49": 10.41706, + "50": 10.40404, + "51": 10.39839, "52": 10.35696, - "53": 10.35712, - "54": 10.31836, - "55": 10.29143, - "56": 10.29525, - "57": 10.28175, - "58": 10.28576, - "59": 10.23425, - "60": 10.23902, - "61": 10.18539, - "62": 10.15955, - "63": 10.21087, - "64": 10.17133, - "65": 10.14708, + "53": 10.35711, + "54": 10.31833, + "55": 10.29141, + "56": 10.29526, + "57": 10.28173, + "58": 10.28575, + "59": 10.23424, + "60": 10.239, + "61": 10.18538, + "62": 10.15953, + "63": 10.21086, + "64": 10.17131, + "65": 10.14707, "66": 10.15528, - "67": 10.13213, - "68": 10.09564, + "67": 10.13211, + "68": 10.09562, "69": 10.11243, - "70": 10.08369, - "71": 10.10364, - "72": 10.08994, - "73": 10.08361, + "70": 10.08366, + "71": 10.10363, + "72": 10.08995, + "73": 10.08359, "74": 10.0635, - "75": 10.04314, - "76": 10.06399, - "77": 10.05648, - "78": 10.01452, - "79": 10.02446, - "80": 10.04024, - "81": 10.05974, - "82": 9.99072, - "83": 9.97283, + "75": 10.04313, + "76": 10.064, + "77": 10.05647, + "78": 10.01453, + "79": 10.02445, + "80": 10.04023, + "81": 10.05971, + "82": 9.99073, + "83": 9.97282, "84": 9.90914, - "85": 9.91074, + "85": 9.91073, "86": 9.99152, - "87": 9.9971, + "87": 9.99709, "88": 9.98034, - "89": 9.92164, + "89": 9.92163, "90": 9.91678, - "91": 9.95049, - "92": 9.92022, - "93": 9.85907, - "94": 9.9275, - "95": 9.92325, - "96": 9.8979, - "97": 9.853, - "98": 9.87376, + "91": 9.95046, + "92": 9.92018, + "93": 9.85906, + "94": 9.92749, + "95": 9.92324, + "96": 9.89787, + "97": 9.85298, + "98": 9.87373, "99": 9.91362, - "100": 9.81531 + "100": 9.81529 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1465368064.0, - "2": 1465368576.0, - "3": 1465368576.0, - "4": 1465368576.0, - "5": 1465368576.0, - "6": 1465368576.0, - "7": 1465368576.0, - "8": 1465368576.0, - "9": 1465368576.0, - "10": 1465368576.0, - "11": 1465368576.0, - "12": 1465368576.0, - "13": 1465368576.0, - "14": 1465368576.0, - "15": 1465368576.0, - "16": 1465368576.0, - "17": 1597485568.0, - "18": 1597485568.0, - "19": 1597485568.0, - "20": 1597485568.0, - "21": 1597485568.0, - "22": 1597485568.0, - "23": 1597485568.0, - "24": 1597485568.0, - "25": 1597485568.0, - "26": 1597485568.0, - "27": 1597485568.0, - "28": 1597485568.0, - "29": 1597485568.0, - "30": 1597485568.0, - "31": 1597485568.0, - "32": 1597485568.0, - "33": 1597485568.0, - "34": 1597485568.0, - "35": 1597485568.0, - "36": 1597485568.0, - "37": 1597485568.0, - "38": 1597485568.0, - "39": 1597485568.0, - "40": 1597485568.0, - "41": 1597485568.0, - "42": 1597485568.0, - "43": 1597485568.0, - "44": 1597485568.0, - "45": 1597485568.0, - "46": 1597485568.0, - "47": 1597485568.0, - "48": 1597485568.0, - "49": 1597485568.0, - "50": 1597485568.0, - "51": 1597485568.0, - "52": 1597485568.0, - "53": 1597485568.0, - "54": 1597485568.0, - "55": 1597485568.0, - "56": 1597485568.0, - "57": 1597485568.0, - "58": 1597485568.0, - "59": 1597485568.0, - "60": 1597485568.0, - "61": 1597485568.0, - "62": 1597485568.0, - "63": 1597485568.0, - "64": 1597485568.0, - "65": 1597485568.0, - "66": 1597485568.0, - "67": 1597485568.0, - "68": 1597485568.0, - "69": 1597485568.0, - "70": 1597485568.0, - "71": 1597485568.0, - "72": 1597485568.0, - "73": 1597485568.0, - "74": 1597485568.0, - "75": 1597485568.0, - "76": 1597485568.0, - "77": 1597485568.0, - "78": 1597485568.0, - "79": 1597485568.0, - "80": 1597485568.0, - "81": 1597485568.0, - "82": 1597485568.0, - "83": 1597485568.0, - "84": 1597485568.0, - "85": 1597485568.0, - "86": 1597485568.0, - "87": 1597485568.0, - "88": 1597485568.0, - "89": 1597485568.0, - "90": 1597485568.0, - "91": 1597485568.0, - "92": 1597485568.0, - "93": 1597485568.0, - "94": 1597485568.0, - "95": 1597485568.0, - "96": 1597485568.0, - "97": 1597485568.0, - "98": 1597485568.0, - "99": 1597485568.0, - "100": 1597485568.0 + "1": 1464319488.0, + "2": 1464320000.0, + "3": 1464320000.0, + "4": 1464320000.0, + "5": 1464320000.0, + "6": 1464320000.0, + "7": 1464320000.0, + "8": 1464320000.0, + "9": 1464320000.0, + "10": 1464320000.0, + "11": 1464320000.0, + "12": 1464320000.0, + "13": 1464320000.0, + "14": 1464320000.0, + "15": 1464320000.0, + "16": 1464320000.0, + "17": 1597092352.0, + "18": 1597092352.0, + "19": 1597092352.0, + "20": 1597092352.0, + "21": 1597092352.0, + "22": 1597092352.0, + "23": 1597092352.0, + "24": 1597092352.0, + "25": 1597092352.0, + "26": 1597092352.0, + "27": 1597092352.0, + "28": 1597092352.0, + "29": 1597092352.0, + "30": 1597092352.0, + "31": 1597092352.0, + "32": 1597092352.0, + "33": 1597092352.0, + "34": 1597092352.0, + "35": 1597092352.0, + "36": 1597092352.0, + "37": 1597092352.0, + "38": 1597092352.0, + "39": 1597092352.0, + "40": 1597092352.0, + "41": 1597092352.0, + "42": 1597092352.0, + "43": 1597092352.0, + "44": 1597092352.0, + "45": 1597092352.0, + "46": 1597092352.0, + "47": 1597092352.0, + "48": 1597092352.0, + "49": 1597092352.0, + "50": 1597092352.0, + "51": 1597092352.0, + "52": 1597092352.0, + "53": 1597092352.0, + "54": 1597092352.0, + "55": 1597092352.0, + "56": 1597092352.0, + "57": 1597092352.0, + "58": 1597092352.0, + "59": 1597092352.0, + "60": 1597092352.0, + "61": 1597092352.0, + "62": 1597092352.0, + "63": 1597092352.0, + "64": 1597092352.0, + "65": 1597092352.0, + "66": 1597092352.0, + "67": 1597092352.0, + "68": 1597092352.0, + "69": 1597092352.0, + "70": 1597092352.0, + "71": 1597092352.0, + "72": 1597092352.0, + "73": 1597092352.0, + "74": 1597092352.0, + "75": 1597092352.0, + "76": 1597092352.0, + "77": 1597092352.0, + "78": 1597092352.0, + "79": 1597092352.0, + "80": 1597092352.0, + "81": 1597092352.0, + "82": 1597092352.0, + "83": 1597092352.0, + "84": 1597092352.0, + "85": 1597092352.0, + "86": 1597092352.0, + "87": 1597092352.0, + "88": 1597092352.0, + "89": 1597092352.0, + "90": 1597092352.0, + "91": 1597092352.0, + "92": 1597092352.0, + "93": 1597092352.0, + "94": 1597092352.0, + "95": 1597092352.0, + "96": 1597092352.0, + "97": 1597092352.0, + "98": 1597092352.0, + "99": 1597092352.0, + "100": 1597092352.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.57064, - "3": 0.32818, - "4": 0.31467, - "5": 0.31563, - "6": 0.31459, - "7": 0.31641, - "8": 0.31542, - "9": 0.31522, - "10": 0.31401, - "11": 0.31473, - "12": 0.3137, - "13": 0.31569, - "14": 0.31508, - "15": 0.31573, - "16": 0.45482, - "17": 0.33936, - "18": 0.33302, - "19": 0.33519, - "20": 0.33487, - "21": 0.33441, - "22": 0.33409, - "23": 0.33427, - "24": 0.33457, - "25": 0.33193, - "26": 0.33274, - "27": 0.3334, - "28": 0.33422, - "29": 0.3317, - "30": 0.3312, - "31": 0.3326, - "32": 0.33174, - "33": 0.33107, - "34": 0.33094, - "35": 0.33407, - "36": 0.33342, - "37": 0.33149, - "38": 0.33032, - "39": 0.32865, - "40": 0.33216, - "41": 0.33172, - "42": 0.33404, - "43": 0.33411, - "44": 0.33584, - "45": 0.33468, - "46": 0.33459, - "47": 0.33575, - "48": 0.36327, - "49": 0.33503, - "50": 0.33611, - "51": 0.49557, - "52": 0.38374, - "53": 0.33677, - "54": 0.33767, - "55": 0.33167, - "56": 0.33305, - "57": 0.33414, - "58": 0.33221, - "59": 0.33404, - "60": 0.33319, - "61": 0.35463, - "62": 0.40329, - "63": 0.34372, - "64": 0.333, - "65": 0.33199, - "66": 0.33589, - "67": 0.33639, - "68": 0.33614, - "69": 0.33684, - "70": 0.33528, - "71": 0.33357, - "72": 0.33359, - "73": 0.33345, - "74": 0.33505, - "75": 0.33517, - "76": 0.33487, - "77": 0.33748, - "78": 0.33525, - "79": 0.3377, - "80": 0.33467, - "81": 0.33608, - "82": 0.33827, - "83": 0.35925, - "84": 0.33697, - "85": 0.33119, - "86": 0.33224, - "87": 0.33237, - "88": 0.33168, - "89": 0.33133, - "90": 0.33353, - "91": 0.3351, - "92": 0.33418, - "93": 0.33525, - "94": 0.33571, - "95": 0.33515, - "96": 0.33529, - "97": 0.33522, - "98": 0.33507, - "99": 0.33543, - "100": 0.33605 + "2": 4.76205, + "3": 0.29645, + "4": 0.26127, + "5": 0.26231, + "6": 0.26184, + "7": 0.26131, + "8": 0.2612, + "9": 0.26397, + "10": 0.26271, + "11": 0.26433, + "12": 0.26358, + "13": 0.26418, + "14": 0.26534, + "15": 0.26604, + "16": 0.42937, + "17": 0.28099, + "18": 0.27789, + "19": 0.27835, + "20": 0.27917, + "21": 0.27711, + "22": 0.27891, + "23": 0.27935, + "24": 0.27957, + "25": 0.27966, + "26": 0.27744, + "27": 0.27804, + "28": 0.27957, + "29": 0.27805, + "30": 0.27838, + "31": 0.28542, + "32": 0.28038, + "33": 0.28108, + "34": 0.28135, + "35": 0.28146, + "36": 0.28019, + "37": 0.2766, + "38": 0.28001, + "39": 0.27997, + "40": 0.27835, + "41": 0.28149, + "42": 0.27794, + "43": 0.27769, + "44": 0.27809, + "45": 0.27804, + "46": 0.2782, + "47": 0.27663, + "48": 0.2791, + "49": 0.28015, + "50": 0.27841, + "51": 0.44019, + "52": 0.33223, + "53": 0.27896, + "54": 0.28152, + "55": 0.28017, + "56": 0.28055, + "57": 0.28412, + "58": 0.27746, + "59": 0.27893, + "60": 0.28132, + "61": 0.27915, + "62": 0.28067, + "63": 0.28132, + "64": 0.28194, + "65": 0.27908, + "66": 0.281, + "67": 0.28014, + "68": 0.28151, + "69": 0.28189, + "70": 0.28082, + "71": 0.28102, + "72": 0.28443, + "73": 0.28028, + "74": 0.27801, + "75": 0.27963, + "76": 0.27885, + "77": 0.27764, + "78": 0.2775, + "79": 0.27858, + "80": 0.27799, + "81": 0.27841, + "82": 0.27688, + "83": 0.27871, + "84": 0.28132, + "85": 0.2777, + "86": 0.27611, + "87": 0.27812, + "88": 0.27913, + "89": 0.28008, + "90": 0.27801, + "91": 0.27747, + "92": 0.27836, + "93": 0.28122, + "94": 0.2774, + "95": 0.27858, + "96": 0.2772, + "97": 0.27828, + "98": 0.27841, + "99": 0.27928, + "100": 0.28001 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2159.0, - "17": 2560.0, - "18": 2249.0, - "19": 2176.0, - "20": 2256.0, - "21": 2385.0, - "22": 2063.0, - "23": 1983.0, - "24": 2238.0, - "25": 1971.0, - "26": 2208.0, - "27": 2282.0, - "28": 2223.0, - "29": 2238.0, - "30": 1973.0, - "31": 2425.0, - "32": 2295.0, - "33": 2048.0, - "34": 2303.0, - "35": 2268.0, - "36": 2224.0, - "37": 2273.0, - "38": 2550.0, - "39": 2326.0, - "40": 2239.0, - "41": 2323.0, - "42": 2008.0, - "43": 2580.0, - "44": 2073.0, - "45": 2678.0, - "46": 2212.0, - "47": 2351.0, - "48": 2373.0, - "49": 2572.0, - "50": 2384.0, - "51": 2145.0, - "52": 2552.0, - "53": 2588.0, - "54": 2412.0, - "55": 2353.0, - "56": 2664.0, - "57": 2263.0, - "58": 3299.0, - "59": 2567.0, - "60": 3005.0, - "61": 2438.0, - "62": 3160.0, - "63": 3270.0, - "64": 2785.0, - "65": 2938.0, - "66": 3077.0, - "67": 3347.0, - "68": 3337.0, - "69": 3026.0, - "70": 3635.0, - "71": 2916.0, - "72": 2982.0, - "73": 3405.0, - "74": 3143.0, - "75": 3874.0, - "76": 3635.0, - "77": 3466.0, - "78": 3688.0, - "79": 5543.0, - "80": 3722.0, - "81": 4065.0, - "82": 2887.0, - "83": 3352.0, - "84": 3929.0, - "85": 4056.0, - "86": 3472.0, - "87": 2663.0, - "88": 4257.0, - "89": 4063.0, - "90": 5060.0, - "91": 2917.0, - "92": 3977.0, - "93": 3920.0, - "94": 3284.0, - "95": 3709.0, - "96": 4570.0, - "97": 4467.0, - "98": 4380.0, - "99": 3448.0, - "100": 3807.0 + "16": 2090.0, + "17": 2642.0, + "18": 2323.0, + "19": 2072.0, + "20": 2358.0, + "21": 2376.0, + "22": 2082.0, + "23": 2046.0, + "24": 2235.0, + "25": 1922.0, + "26": 2167.0, + "27": 2220.0, + "28": 2332.0, + "29": 2472.0, + "30": 1988.0, + "31": 2497.0, + "32": 2266.0, + "33": 2105.0, + "34": 2329.0, + "35": 2275.0, + "36": 2211.0, + "37": 2270.0, + "38": 2454.0, + "39": 2403.0, + "40": 2248.0, + "41": 2288.0, + "42": 2035.0, + "43": 2514.0, + "44": 2137.0, + "45": 2595.0, + "46": 2270.0, + "47": 2250.0, + "48": 2392.0, + "49": 2483.0, + "50": 2450.0, + "51": 2299.0, + "52": 2579.0, + "53": 2494.0, + "54": 2311.0, + "55": 2398.0, + "56": 2637.0, + "57": 2314.0, + "58": 3264.0, + "59": 2662.0, + "60": 2967.0, + "61": 2462.0, + "62": 3097.0, + "63": 3247.0, + "64": 2926.0, + "65": 2977.0, + "66": 3103.0, + "67": 3364.0, + "68": 3387.0, + "69": 2975.0, + "70": 3809.0, + "71": 2902.0, + "72": 2994.0, + "73": 3386.0, + "74": 3231.0, + "75": 3811.0, + "76": 3648.0, + "77": 3489.0, + "78": 3550.0, + "79": 5447.0, + "80": 3663.0, + "81": 4094.0, + "82": 2990.0, + "83": 3287.0, + "84": 3784.0, + "85": 4049.0, + "86": 3425.0, + "87": 2647.0, + "88": 4355.0, + "89": 4137.0, + "90": 5098.0, + "91": 2799.0, + "92": 3843.0, + "93": 3980.0, + "94": 3246.0, + "95": 3807.0, + "96": 4497.0, + "97": 4540.0, + "98": 4336.0, + "99": 3437.0, + "100": 3918.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json index 78da715fca6..990aad9f757 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json @@ -4,104 +4,104 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89757, - "2": 10.89076, - "3": 10.88414, - "4": 10.89075, + "1": 10.89758, + "2": 10.89074, + "3": 10.88416, + "4": 10.89076, "5": 10.88375, - "6": 10.88793, - "7": 10.88856, - "8": 10.88592, - "9": 10.88899, - "10": 10.8852, - "11": 10.89078, + "6": 10.88788, + "7": 10.88855, + "8": 10.88593, + "9": 10.88903, + "10": 10.88521, + "11": 10.8908, "12": 10.89125, - "13": 10.88783, - "14": 10.88523, + "13": 10.88782, + "14": 10.88524, "15": 10.88844, - "16": 10.88579, - "17": 10.88587, - "18": 10.88812, + "16": 10.88577, + "17": 10.88588, + "18": 10.88813, "19": 10.89138, - "20": 10.88198, - "21": 10.88925, - "22": 10.875, + "20": 10.88194, + "21": 10.88923, + "22": 10.87496, "23": 10.86547, - "24": 10.86171, - "25": 10.86263, - "26": 10.83462, - "27": 10.83634, - "28": 10.82662, - "29": 10.81094, + "24": 10.86172, + "25": 10.86267, + "26": 10.8346, + "27": 10.83637, + "28": 10.8266, + "29": 10.81092, "30": 10.78664, - "31": 10.78372, - "32": 10.77579, - "33": 10.7342, + "31": 10.78369, + "32": 10.7758, + "33": 10.73418, "34": 10.71119, - "35": 10.70532, - "36": 10.69049, - "37": 10.65866, - "38": 10.64974, - "39": 10.62381, - "40": 10.6069, + "35": 10.70535, + "36": 10.6905, + "37": 10.65867, + "38": 10.64971, + "39": 10.62382, + "40": 10.60691, "41": 10.57833, - "42": 10.5603, - "43": 10.54222, + "42": 10.56029, + "43": 10.54219, "44": 10.5188, - "45": 10.5053, - "46": 10.48038, - "47": 10.46032, - "48": 10.41509, - "49": 10.41634, - "50": 10.40081, - "51": 10.39365, - "52": 10.34945, + "45": 10.50527, + "46": 10.4804, + "47": 10.46031, + "48": 10.41508, + "49": 10.41632, + "50": 10.4008, + "51": 10.39364, + "52": 10.34944, "53": 10.342, - "54": 10.31227, - "55": 10.29325, + "54": 10.31225, + "55": 10.29323, "56": 10.29329, - "57": 10.2813, - "58": 10.28129, - "59": 10.21997, + "57": 10.28126, + "58": 10.28128, + "59": 10.21996, "60": 10.23018, - "61": 10.18393, + "61": 10.18391, "62": 10.15993, "63": 10.20602, - "64": 10.16203, + "64": 10.16204, "65": 10.12249, "66": 10.15236, - "67": 10.12965, - "68": 10.09494, + "67": 10.12963, + "68": 10.09493, "69": 10.10681, - "70": 10.08342, - "71": 10.09959, - "72": 10.08977, - "73": 10.07514, + "70": 10.0834, + "71": 10.09957, + "72": 10.08974, + "73": 10.07511, "74": 10.06651, - "75": 10.03809, - "76": 10.05237, - "77": 10.05049, - "78": 10.00922, - "79": 10.01367, - "80": 10.03041, - "81": 10.05065, - "82": 9.98611, - "83": 9.96212, - "84": 9.89756, + "75": 10.03807, + "76": 10.05239, + "77": 10.05048, + "78": 10.00921, + "79": 10.01364, + "80": 10.03039, + "81": 10.05063, + "82": 9.98609, + "83": 9.96211, + "84": 9.89755, "85": 9.89373, "86": 9.9881, "87": 9.99417, - "88": 9.97617, - "89": 9.92167, - "90": 9.91508, - "91": 9.94041, - "92": 9.91744, - "93": 9.85398, - "94": 9.92293, - "95": 9.90966, + "88": 9.97615, + "89": 9.92166, + "90": 9.91507, + "91": 9.94039, + "92": 9.91741, + "93": 9.85395, + "94": 9.92292, + "95": 9.90964, "96": 9.88828, - "97": 9.84573, - "98": 9.86176, + "97": 9.84571, + "98": 9.86174, "99": 9.91044, "100": 9.80688 } @@ -126,91 +126,91 @@ "13": 284527616.0, "14": 284527616.0, "15": 284527616.0, - "16": 416513536.0, - "17": 416513536.0, - "18": 416513536.0, - "19": 416513536.0, - "20": 416513536.0, - "21": 416513536.0, - "22": 416513536.0, - "23": 416513536.0, - "24": 416513536.0, - "25": 416513536.0, - "26": 416513536.0, - "27": 416513536.0, - "28": 416513536.0, - "29": 416513536.0, - "30": 416513536.0, - "31": 416513536.0, - "32": 416513536.0, - "33": 416513536.0, - "34": 416513536.0, - "35": 416513536.0, - "36": 416513536.0, - "37": 416513536.0, - "38": 416513536.0, - "39": 416513536.0, - "40": 416513536.0, - "41": 416513536.0, - "42": 416513536.0, - "43": 416513536.0, - "44": 416513536.0, - "45": 416513536.0, - "46": 416513536.0, - "47": 416513536.0, - "48": 416513536.0, - "49": 416513536.0, - "50": 416513536.0, - "51": 416513536.0, - "52": 416513536.0, - "53": 416513536.0, - "54": 416513536.0, - "55": 416513536.0, - "56": 416513536.0, - "57": 416513536.0, - "58": 416513536.0, - "59": 416513536.0, - "60": 416513536.0, - "61": 416513536.0, - "62": 416513536.0, - "63": 416513536.0, - "64": 416513536.0, - "65": 416513536.0, - "66": 416513536.0, - "67": 416513536.0, - "68": 416513536.0, - "69": 416513536.0, - "70": 416513536.0, - "71": 416513536.0, - "72": 416513536.0, - "73": 416513536.0, - "74": 416513536.0, - "75": 416513536.0, - "76": 416513536.0, - "77": 416513536.0, - "78": 416513536.0, - "79": 416513536.0, - "80": 416513536.0, - "81": 416513536.0, - "82": 416513536.0, - "83": 416513536.0, - "84": 416513536.0, - "85": 416513536.0, - "86": 416513536.0, - "87": 416513536.0, - "88": 416513536.0, - "89": 416513536.0, - "90": 416513536.0, - "91": 416513536.0, - "92": 416513536.0, - "93": 416513536.0, - "94": 416513536.0, - "95": 416513536.0, - "96": 416513536.0, - "97": 416513536.0, - "98": 416513536.0, - "99": 416513536.0, - "100": 416513536.0 + "16": 417037824.0, + "17": 417037824.0, + "18": 417037824.0, + "19": 417037824.0, + "20": 417037824.0, + "21": 417037824.0, + "22": 417037824.0, + "23": 417037824.0, + "24": 417037824.0, + "25": 417037824.0, + "26": 417037824.0, + "27": 417037824.0, + "28": 417037824.0, + "29": 417037824.0, + "30": 417037824.0, + "31": 417037824.0, + "32": 417037824.0, + "33": 417037824.0, + "34": 417037824.0, + "35": 417037824.0, + "36": 417037824.0, + "37": 417037824.0, + "38": 417037824.0, + "39": 417037824.0, + "40": 417037824.0, + "41": 417037824.0, + "42": 417037824.0, + "43": 417037824.0, + "44": 417037824.0, + "45": 417037824.0, + "46": 417037824.0, + "47": 417037824.0, + "48": 417037824.0, + "49": 417037824.0, + "50": 417037824.0, + "51": 417037824.0, + "52": 417037824.0, + "53": 417037824.0, + "54": 417037824.0, + "55": 417037824.0, + "56": 417037824.0, + "57": 417037824.0, + "58": 417037824.0, + "59": 417037824.0, + "60": 417037824.0, + "61": 417037824.0, + "62": 417037824.0, + "63": 417037824.0, + "64": 417037824.0, + "65": 417037824.0, + "66": 417037824.0, + "67": 417037824.0, + "68": 417037824.0, + "69": 417037824.0, + "70": 417037824.0, + "71": 417037824.0, + "72": 417037824.0, + "73": 417037824.0, + "74": 417037824.0, + "75": 417037824.0, + "76": 417037824.0, + "77": 417037824.0, + "78": 417037824.0, + "79": 417037824.0, + "80": 417037824.0, + "81": 417037824.0, + "82": 417037824.0, + "83": 417037824.0, + "84": 417037824.0, + "85": 417037824.0, + "86": 417037824.0, + "87": 417037824.0, + "88": 417037824.0, + "89": 417037824.0, + "90": 417037824.0, + "91": 417037824.0, + "92": 417037824.0, + "93": 417037824.0, + "94": 417037824.0, + "95": 417037824.0, + "96": 417037824.0, + "97": 417037824.0, + "98": 417037824.0, + "99": 417037824.0, + "100": 417037824.0 } }, "mem-max-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1465368064.0, - "2": 1465368576.0, - "3": 1465368576.0, - "4": 1465368576.0, - "5": 1465368576.0, - "6": 1465368576.0, - "7": 1465368576.0, - "8": 1465368576.0, - "9": 1465368576.0, - "10": 1465368576.0, - "11": 1465368576.0, - "12": 1465368576.0, - "13": 1465368576.0, - "14": 1465368576.0, - "15": 1465368576.0, - "16": 1465368576.0, - "17": 1597485568.0, - "18": 1597485568.0, - "19": 1597485568.0, - "20": 1597485568.0, - "21": 1597485568.0, - "22": 1597485568.0, - "23": 1597485568.0, - "24": 1597485568.0, - "25": 1597485568.0, - "26": 1597485568.0, - "27": 1597485568.0, - "28": 1597485568.0, - "29": 1597485568.0, - "30": 1597485568.0, - "31": 1597485568.0, - "32": 1597485568.0, - "33": 1597485568.0, - "34": 1597485568.0, - "35": 1597485568.0, - "36": 1597485568.0, - "37": 1597485568.0, - "38": 1597485568.0, - "39": 1597485568.0, - "40": 1597485568.0, - "41": 1597485568.0, - "42": 1597485568.0, - "43": 1597485568.0, - "44": 1597485568.0, - "45": 1597485568.0, - "46": 1597485568.0, - "47": 1597485568.0, - "48": 1597485568.0, - "49": 1597485568.0, - "50": 1597485568.0, - "51": 1597485568.0, - "52": 1597485568.0, - "53": 1597485568.0, - "54": 1597485568.0, - "55": 1597485568.0, - "56": 1597485568.0, - "57": 1597485568.0, - "58": 1597485568.0, - "59": 1597485568.0, - "60": 1597485568.0, - "61": 1597485568.0, - "62": 1597485568.0, - "63": 1597485568.0, - "64": 1597485568.0, - "65": 1597485568.0, - "66": 1597485568.0, - "67": 1597485568.0, - "68": 1597485568.0, - "69": 1597485568.0, - "70": 1597485568.0, - "71": 1597485568.0, - "72": 1597485568.0, - "73": 1597485568.0, - "74": 1597485568.0, - "75": 1597485568.0, - "76": 1597485568.0, - "77": 1597485568.0, - "78": 1597485568.0, - "79": 1597485568.0, - "80": 1597485568.0, - "81": 1597485568.0, - "82": 1597485568.0, - "83": 1597485568.0, - "84": 1597485568.0, - "85": 1597485568.0, - "86": 1597485568.0, - "87": 1597485568.0, - "88": 1597485568.0, - "89": 1597485568.0, - "90": 1597485568.0, - "91": 1597485568.0, - "92": 1597485568.0, - "93": 1597485568.0, - "94": 1597485568.0, - "95": 1597485568.0, - "96": 1597485568.0, - "97": 1597485568.0, - "98": 1597485568.0, - "99": 1597485568.0, - "100": 1597485568.0 + "1": 1464319488.0, + "2": 1464320000.0, + "3": 1464320000.0, + "4": 1464320000.0, + "5": 1464320000.0, + "6": 1464320000.0, + "7": 1464320000.0, + "8": 1464320000.0, + "9": 1464320000.0, + "10": 1464320000.0, + "11": 1464320000.0, + "12": 1464320000.0, + "13": 1464320000.0, + "14": 1464320000.0, + "15": 1464320000.0, + "16": 1464320000.0, + "17": 1597616640.0, + "18": 1597616640.0, + "19": 1597616640.0, + "20": 1597616640.0, + "21": 1597616640.0, + "22": 1597616640.0, + "23": 1597616640.0, + "24": 1597616640.0, + "25": 1597616640.0, + "26": 1597616640.0, + "27": 1597616640.0, + "28": 1597616640.0, + "29": 1597616640.0, + "30": 1597616640.0, + "31": 1597616640.0, + "32": 1597616640.0, + "33": 1597616640.0, + "34": 1597616640.0, + "35": 1597616640.0, + "36": 1597616640.0, + "37": 1597616640.0, + "38": 1597616640.0, + "39": 1597616640.0, + "40": 1597616640.0, + "41": 1597616640.0, + "42": 1597616640.0, + "43": 1597616640.0, + "44": 1597616640.0, + "45": 1597616640.0, + "46": 1597616640.0, + "47": 1597616640.0, + "48": 1597616640.0, + "49": 1597616640.0, + "50": 1597616640.0, + "51": 1598662656.0, + "52": 1598662656.0, + "53": 1598662656.0, + "54": 1598662656.0, + "55": 1598662656.0, + "56": 1598662656.0, + "57": 1598662656.0, + "58": 1598662656.0, + "59": 1598662656.0, + "60": 1598662656.0, + "61": 1598662656.0, + "62": 1598662656.0, + "63": 1598662656.0, + "64": 1598662656.0, + "65": 1598662656.0, + "66": 1598662656.0, + "67": 1598662656.0, + "68": 1598662656.0, + "69": 1598662656.0, + "70": 1598662656.0, + "71": 1598662656.0, + "72": 1598662656.0, + "73": 1598662656.0, + "74": 1598662656.0, + "75": 1598662656.0, + "76": 1598662656.0, + "77": 1598662656.0, + "78": 1598662656.0, + "79": 1598662656.0, + "80": 1598662656.0, + "81": 1598662656.0, + "82": 1598662656.0, + "83": 1598662656.0, + "84": 1598662656.0, + "85": 1598662656.0, + "86": 1598662656.0, + "87": 1598662656.0, + "88": 1598662656.0, + "89": 1598662656.0, + "90": 1598662656.0, + "91": 1598662656.0, + "92": 1598662656.0, + "93": 1598662656.0, + "94": 1598662656.0, + "95": 1598662656.0, + "96": 1598662656.0, + "97": 1598662656.0, + "98": 1598662656.0, + "99": 1598662656.0, + "100": 1598662656.0 } }, "iteration-time": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.5325, - "3": 0.20612, - "4": 0.18723, - "5": 0.18752, - "6": 0.18748, - "7": 0.18695, - "8": 0.18714, - "9": 0.18673, - "10": 0.18733, - "11": 0.18728, - "12": 0.18725, - "13": 0.1817, - "14": 0.18286, - "15": 0.1825, - "16": 0.27285, - "17": 0.23025, - "18": 0.19014, - "19": 0.1911, - "20": 0.18961, - "21": 0.18907, - "22": 0.19019, - "23": 0.18957, - "24": 0.18918, - "25": 0.18919, - "26": 0.18906, - "27": 0.18893, - "28": 0.19114, - "29": 0.18931, - "30": 0.18919, - "31": 0.18902, - "32": 0.1887, - "33": 0.18999, - "34": 0.19229, - "35": 0.18922, - "36": 0.18914, - "37": 0.1986, - "38": 0.22972, - "39": 0.19331, - "40": 0.19018, - "41": 0.1895, - "42": 0.18887, - "43": 0.18941, - "44": 0.19087, - "45": 0.18967, - "46": 0.18958, - "47": 0.18912, - "48": 0.18932, - "49": 0.1904, - "50": 0.18919, - "51": 0.25672, - "52": 0.23046, - "53": 0.19031, - "54": 0.18985, - "55": 0.18928, - "56": 0.18926, - "57": 0.19181, - "58": 0.18981, - "59": 0.18934, - "60": 0.19007, - "61": 0.19114, - "62": 0.19179, - "63": 0.19019, - "64": 0.1892, - "65": 0.18882, - "66": 0.18898, - "67": 0.18919, - "68": 0.18988, - "69": 0.18966, - "70": 0.18969, - "71": 0.18958, - "72": 0.18881, - "73": 0.18939, - "74": 0.18895, - "75": 0.18904, - "76": 0.18958, - "77": 0.18896, - "78": 0.18967, - "79": 0.18983, - "80": 0.18896, - "81": 0.18884, - "82": 0.18861, - "83": 0.18919, - "84": 0.19241, - "85": 0.18948, - "86": 0.18871, - "87": 0.18894, - "88": 0.18917, - "89": 0.18925, - "90": 0.18957, - "91": 0.18879, - "92": 0.18919, - "93": 0.19301, - "94": 0.19204, - "95": 0.19124, - "96": 0.18922, - "97": 0.19096, - "98": 0.18903, - "99": 0.19095, - "100": 0.19052 + "2": 4.21341, + "3": 0.22288, + "4": 0.20226, + "5": 0.20248, + "6": 0.20195, + "7": 0.20487, + "8": 0.20377, + "9": 0.20315, + "10": 0.20276, + "11": 0.20197, + "12": 0.19633, + "13": 0.19621, + "14": 0.19642, + "15": 0.19556, + "16": 0.28086, + "17": 0.20704, + "18": 0.2045, + "19": 0.20509, + "20": 0.20448, + "21": 0.20612, + "22": 0.20382, + "23": 0.20448, + "24": 0.20844, + "25": 0.2021, + "26": 0.20206, + "27": 0.20125, + "28": 0.20111, + "29": 0.20232, + "30": 0.20099, + "31": 0.20166, + "32": 0.20339, + "33": 0.20093, + "34": 0.2016, + "35": 0.20085, + "36": 0.20608, + "37": 0.20451, + "38": 0.20128, + "39": 0.20378, + "40": 0.20498, + "41": 0.21176, + "42": 0.21114, + "43": 0.21179, + "44": 0.21145, + "45": 0.21159, + "46": 0.21228, + "47": 0.21241, + "48": 0.21143, + "49": 0.21091, + "50": 0.21143, + "51": 0.35062, + "52": 0.24259, + "53": 0.20341, + "54": 0.20399, + "55": 0.20326, + "56": 0.20268, + "57": 0.20164, + "58": 0.20151, + "59": 0.20131, + "60": 0.20346, + "61": 0.20191, + "62": 0.20193, + "63": 0.20265, + "64": 0.20215, + "65": 0.20223, + "66": 0.20236, + "67": 0.20241, + "68": 0.2028, + "69": 0.20155, + "70": 0.20308, + "71": 0.2021, + "72": 0.20234, + "73": 0.20088, + "74": 0.20157, + "75": 0.20267, + "76": 0.20159, + "77": 0.20182, + "78": 0.20186, + "79": 0.202, + "80": 0.20679, + "81": 0.2017, + "82": 0.20338, + "83": 0.20314, + "84": 0.20353, + "85": 0.20448, + "86": 0.20405, + "87": 0.2058, + "88": 0.20421, + "89": 0.38789, + "90": 0.39269, + "91": 0.20663, + "92": 0.20636, + "93": 0.20683, + "94": 0.20746, + "95": 0.20584, + "96": 0.20438, + "97": 0.20288, + "98": 0.20249, + "99": 0.20266, + "100": 0.20319 } }, "num-zeros": { @@ -447,91 +447,91 @@ "13": "nan", "14": "nan", "15": "nan", - "16": 2012.0, - "17": 2525.0, - "18": 2270.0, - "19": 1958.0, - "20": 2327.0, - "21": 2331.0, - "22": 2160.0, - "23": 2112.0, - "24": 2098.0, - "25": 1958.0, - "26": 2110.0, - "27": 2245.0, - "28": 2284.0, - "29": 2351.0, - "30": 1913.0, - "31": 2467.0, - "32": 2294.0, - "33": 2082.0, - "34": 2307.0, - "35": 2318.0, - "36": 2250.0, - "37": 2177.0, - "38": 2457.0, - "39": 2281.0, - "40": 2255.0, - "41": 2199.0, - "42": 2096.0, - "43": 2430.0, - "44": 2123.0, - "45": 2627.0, - "46": 2254.0, - "47": 2300.0, - "48": 2352.0, - "49": 2469.0, - "50": 2494.0, - "51": 2240.0, - "52": 2577.0, - "53": 2590.0, - "54": 2294.0, - "55": 2263.0, - "56": 2662.0, - "57": 2276.0, - "58": 3221.0, - "59": 2539.0, - "60": 3055.0, - "61": 2438.0, - "62": 2965.0, - "63": 3320.0, - "64": 2754.0, + "16": 1982.0, + "17": 2464.0, + "18": 2254.0, + "19": 2029.0, + "20": 2317.0, + "21": 2275.0, + "22": 2138.0, + "23": 2073.0, + "24": 2157.0, + "25": 1998.0, + "26": 2137.0, + "27": 2237.0, + "28": 2354.0, + "29": 2392.0, + "30": 1955.0, + "31": 2503.0, + "32": 2238.0, + "33": 2130.0, + "34": 2354.0, + "35": 2256.0, + "36": 2217.0, + "37": 2234.0, + "38": 2462.0, + "39": 2264.0, + "40": 2217.0, + "41": 2294.0, + "42": 2136.0, + "43": 2485.0, + "44": 2174.0, + "45": 2605.0, + "46": 2196.0, + "47": 2351.0, + "48": 2332.0, + "49": 2449.0, + "50": 2391.0, + "51": 2232.0, + "52": 2589.0, + "53": 2541.0, + "54": 2326.0, + "55": 2231.0, + "56": 2615.0, + "57": 2232.0, + "58": 3310.0, + "59": 2553.0, + "60": 2945.0, + "61": 2422.0, + "62": 2943.0, + "63": 3285.0, + "64": 2761.0, "65": 2866.0, - "66": 3104.0, - "67": 3322.0, - "68": 3009.0, - "69": 3106.0, - "70": 3656.0, - "71": 2897.0, - "72": 2893.0, - "73": 3254.0, - "74": 3116.0, - "75": 3898.0, - "76": 3522.0, - "77": 3521.0, - "78": 3507.0, - "79": 5478.0, - "80": 3514.0, - "81": 4336.0, - "82": 3007.0, - "83": 3300.0, - "84": 3941.0, - "85": 3973.0, - "86": 3457.0, - "87": 2705.0, - "88": 4224.0, - "89": 4308.0, + "66": 3079.0, + "67": 3245.0, + "68": 3159.0, + "69": 3124.0, + "70": 3627.0, + "71": 2888.0, + "72": 3051.0, + "73": 3295.0, + "74": 3149.0, + "75": 3945.0, + "76": 3549.0, + "77": 3559.0, + "78": 3516.0, + "79": 5398.0, + "80": 3557.0, + "81": 4303.0, + "82": 2965.0, + "83": 3422.0, + "84": 3906.0, + "85": 4042.0, + "86": 3582.0, + "87": 2704.0, + "88": 4189.0, + "89": 4228.0, "90": 5060.0, - "91": 2666.0, - "92": 3783.0, - "93": 3904.0, - "94": 3305.0, - "95": 3553.0, - "96": 4233.0, - "97": 4488.0, - "98": 4417.0, - "99": 3247.0, - "100": 3493.0 + "91": 2720.0, + "92": 3889.0, + "93": 3943.0, + "94": 3315.0, + "95": 3539.0, + "96": 4499.0, + "97": 4487.0, + "98": 4457.0, + "99": 3230.0, + "100": 3424.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json index 2fac2b44381..c7bd25189d0 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 544347136.0, - "2": 544348672.0, - "3": 544348672.0, - "4": 544348672.0, - "5": 544348672.0, - "6": 544348672.0, - "7": 544348672.0, - "8": 544348672.0, - "9": 544348672.0, - "10": 544348672.0, - "11": 544348672.0, - "12": 544348672.0, - "13": 544348672.0, - "14": 544348672.0, - "15": 544348672.0, - "16": 544348672.0, - "17": 544348672.0, - "18": 544348672.0, - "19": 544348672.0, - "20": 544348672.0, - "21": 544348672.0, - "22": 544348672.0, - "23": 544348672.0, - "24": 544348672.0, - "25": 544348672.0, - "26": 544348672.0, - "27": 544348672.0, - "28": 544348672.0, - "29": 544348672.0, - "30": 544348672.0, - "31": 544348672.0, - "32": 544348672.0, - "33": 544348672.0, - "34": 544348672.0, - "35": 544348672.0, - "36": 544348672.0, - "37": 544348672.0, - "38": 544348672.0, - "39": 544348672.0, - "40": 544348672.0, - "41": 544348672.0, - "42": 544348672.0, - "43": 544348672.0, - "44": 544348672.0, - "45": 544348672.0, - "46": 544348672.0, - "47": 544348672.0, - "48": 544348672.0, - "49": 544348672.0, - "50": 544348672.0 + "1": 594416640.0, + "2": 594418176.0, + "3": 594418176.0, + "4": 594418176.0, + "5": 594418176.0, + "6": 594418176.0, + "7": 594418176.0, + "8": 594418176.0, + "9": 594418176.0, + "10": 594418176.0, + "11": 594418176.0, + "12": 594418176.0, + "13": 594418176.0, + "14": 594418176.0, + "15": 594418176.0, + "16": 594418176.0, + "17": 594418176.0, + "18": 594418176.0, + "19": 594418176.0, + "20": 594418176.0, + "21": 594418176.0, + "22": 594418176.0, + "23": 594418176.0, + "24": 594418176.0, + "25": 594418176.0, + "26": 594418176.0, + "27": 594418176.0, + "28": 594418176.0, + "29": 594418176.0, + "30": 594418176.0, + "31": 594418176.0, + "32": 594418176.0, + "33": 594418176.0, + "34": 594418176.0, + "35": 594418176.0, + "36": 594418176.0, + "37": 594418176.0, + "38": 594418176.0, + "39": 594418176.0, + "40": 594418176.0, + "41": 594418176.0, + "42": 594418176.0, + "43": 594418176.0, + "44": 594418176.0, + "45": 594418176.0, + "46": 594418176.0, + "47": 594418176.0, + "48": 594418176.0, + "49": 594418176.0, + "50": 594418176.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2335718400.0, - "2": 2458891776.0, - "3": 2479774208.0, - "4": 2479774208.0, - "5": 2479774208.0, - "6": 2479774208.0, - "7": 2479774208.0, - "8": 2479774208.0, - "9": 2479774208.0, - "10": 2479774208.0, - "11": 2479774208.0, - "12": 2479774208.0, - "13": 2479774208.0, - "14": 2479774208.0, - "15": 2479774208.0, - "16": 2479774208.0, - "17": 2479774208.0, - "18": 2479774208.0, - "19": 2479774208.0, - "20": 2479774208.0, - "21": 2479774208.0, - "22": 2479774208.0, - "23": 2479774208.0, - "24": 2479774208.0, - "25": 2479774208.0, - "26": 2479774208.0, - "27": 2479774208.0, - "28": 2479774208.0, - "29": 2479774208.0, - "30": 2479774208.0, - "31": 2479774208.0, - "32": 2479774208.0, - "33": 2479774208.0, - "34": 2479774208.0, - "35": 2479774208.0, - "36": 2479774208.0, - "37": 2479774208.0, - "38": 2479774208.0, - "39": 2481645568.0, - "40": 2481645568.0, - "41": 2481645568.0, - "42": 2481645568.0, - "43": 2481645568.0, - "44": 2481645568.0, - "45": 2489437184.0, - "46": 2489437184.0, - "47": 2489437184.0, - "48": 2489437184.0, - "49": 2489437184.0, - "50": 2489437184.0 + "1": 2385787904.0, + "2": 2508961280.0, + "3": 2529843712.0, + "4": 2529843712.0, + "5": 2529843712.0, + "6": 2529843712.0, + "7": 2529843712.0, + "8": 2529843712.0, + "9": 2529843712.0, + "10": 2529843712.0, + "11": 2529843712.0, + "12": 2529843712.0, + "13": 2529843712.0, + "14": 2529843712.0, + "15": 2529843712.0, + "16": 2529843712.0, + "17": 2529843712.0, + "18": 2529843712.0, + "19": 2529843712.0, + "20": 2529843712.0, + "21": 2529843712.0, + "22": 2529843712.0, + "23": 2529843712.0, + "24": 2529843712.0, + "25": 2529843712.0, + "26": 2529843712.0, + "27": 2529843712.0, + "28": 2529843712.0, + "29": 2529843712.0, + "30": 2529843712.0, + "31": 2529843712.0, + "32": 2529843712.0, + "33": 2529843712.0, + "34": 2529843712.0, + "35": 2529843712.0, + "36": 2529843712.0, + "37": 2529843712.0, + "38": 2529843712.0, + "39": 2531715072.0, + "40": 2531715072.0, + "41": 2531715072.0, + "42": 2531715072.0, + "43": 2531715072.0, + "44": 2531715072.0, + "45": 2539506688.0, + "46": 2539506688.0, + "47": 2539506688.0, + "48": 2539506688.0, + "49": 2539506688.0, + "50": 2539506688.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 13.12071, - "3": 0.14634, - "4": 0.55256, - "5": 0.74839, - "6": 0.76356, - "7": 0.38394, - "8": 0.50723, - "9": 0.93992, - "10": 0.50174, - "11": 0.67907, - "12": 0.69163, - "13": 0.48596, - "14": 0.52873, - "15": 0.44691, - "16": 0.537, - "17": 0.59548, - "18": 0.8704, - "19": 1.26369, - "20": 0.36655, - "21": 0.39997, - "22": 0.54097, - "23": 1.28904, - "24": 0.72709, - "25": 0.23533, - "26": 0.59206, - "27": 1.13639, - "28": 0.44584, - "29": 0.53564, - "30": 0.70672, - "31": 0.91962, - "32": 0.35207, - "33": 1.0285, - "34": 0.70842, - "35": 0.5221, - "36": 0.99081, - "37": 0.63678, - "38": 0.30022, - "39": 0.69391, - "40": 0.79304, - "41": 0.68078, - "42": 0.56356, - "43": 0.51748, - "44": 0.81076, - "45": 0.98012, - "46": 0.41141, - "47": 0.70899, - "48": 0.68674, - "49": 0.56065, - "50": 1.00066 + "2": 12.54605, + "3": 0.12427, + "4": 0.60177, + "5": 0.90977, + "6": 1.01173, + "7": 0.3394, + "8": 0.46492, + "9": 0.80115, + "10": 0.81693, + "11": 0.57731, + "12": 0.96087, + "13": 0.52499, + "14": 0.62418, + "15": 0.73674, + "16": 0.47358, + "17": 0.76277, + "18": 0.60215, + "19": 0.81429, + "20": 0.50144, + "21": 0.42636, + "22": 1.06813, + "23": 0.84995, + "24": 0.59619, + "25": 0.50497, + "26": 1.388, + "27": 0.51478, + "28": 0.61725, + "29": 0.58964, + "30": 0.84345, + "31": 0.67049, + "32": 0.64298, + "33": 0.81336, + "34": 0.73803, + "35": 0.40901, + "36": 0.90227, + "37": 0.62965, + "38": 0.83055, + "39": 0.57165, + "40": 0.68358, + "41": 0.72135, + "42": 0.40091, + "43": 0.53488, + "44": 0.89523, + "45": 1.10172, + "46": 0.54933, + "47": 0.74879, + "48": 0.66791, + "49": 0.64939, + "50": 0.89175 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json index 9cf1b6ce710..663bc85abaa 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.91731, - "2": 10.90352, - "3": 10.9039, - "4": 10.89709, - "5": 10.91817, - "6": 10.90942, - "7": 10.8988, - "8": 10.89986, - "9": 10.90354, - "10": 10.90854, - "11": 10.8966, - "12": 10.90281, - "13": 10.88297, - "14": 10.88235, - "15": 10.8763, - "16": 10.86721, - "17": 10.86456, - "18": 10.85831, - "19": 10.86348, - "20": 10.81664, - "21": 10.80038, - "22": 10.79097, - "23": 10.78104, - "24": 10.74835, - "25": 10.75884, - "26": 10.73609, - "27": 10.72247, - "28": 10.67482, - "29": 10.63467, - "30": 10.61837, - "31": 10.60469, - "32": 10.58917, - "33": 10.56143, - "34": 10.53557, - "35": 10.52956, - "36": 10.51913, - "37": 10.4827, - "38": 10.48811, - "39": 10.4527, - "40": 10.43573, - "41": 10.41525, - "42": 10.40633, - "43": 10.38815, - "44": 10.3424, - "45": 10.36225, - "46": 10.32579, - "47": 10.30532, - "48": 10.27376, - "49": 10.26191, - "50": 10.26168, - "51": 10.25947, - "52": 10.21672, - "53": 10.22296, - "54": 10.1793, - "55": 10.16541, - "56": 10.17769, - "57": 10.16912, - "58": 10.17691, - "59": 10.13, - "60": 10.14422, - "61": 10.09549, - "62": 10.06975, - "63": 10.12727, - "64": 10.09506, - "65": 10.07407, - "66": 10.0836, - "67": 10.06391, - "68": 10.0268, - "69": 10.0513, - "70": 10.02716, - "71": 10.04647, + "1": 10.91589, + "2": 10.90332, + "3": 10.90347, + "4": 10.89774, + "5": 10.9187, + "6": 10.90932, + "7": 10.89761, + "8": 10.90061, + "9": 10.9039, + "10": 10.90834, + "11": 10.89574, + "12": 10.90275, + "13": 10.88282, + "14": 10.88306, + "15": 10.87697, + "16": 10.86788, + "17": 10.86436, + "18": 10.85948, + "19": 10.86322, + "20": 10.81598, + "21": 10.80064, + "22": 10.79033, + "23": 10.78083, + "24": 10.74886, + "25": 10.75938, + "26": 10.73731, + "27": 10.72307, + "28": 10.67446, + "29": 10.63463, + "30": 10.61775, + "31": 10.60463, + "32": 10.58936, + "33": 10.56185, + "34": 10.53587, + "35": 10.52959, + "36": 10.51926, + "37": 10.48229, + "38": 10.48784, + "39": 10.45258, + "40": 10.43514, + "41": 10.41533, + "42": 10.40609, + "43": 10.38854, + "44": 10.34231, + "45": 10.36248, + "46": 10.32544, + "47": 10.30585, + "48": 10.27412, + "49": 10.26208, + "50": 10.26192, + "51": 10.26006, + "52": 10.21685, + "53": 10.22334, + "54": 10.17948, + "55": 10.16584, + "56": 10.17775, + "57": 10.16954, + "58": 10.17712, + "59": 10.13005, + "60": 10.14441, + "61": 10.09527, + "62": 10.0697, + "63": 10.12716, + "64": 10.09497, + "65": 10.07382, + "66": 10.08357, + "67": 10.06349, + "68": 10.02719, + "69": 10.05119, + "70": 10.02734, + "71": 10.04629, "72": 10.03295, - "73": 10.02457, - "74": 10.00647, - "75": 9.98663, - "76": 10.00586, - "77": 10.00663, - "78": 9.95709, - "79": 9.96949, - "80": 9.98359, - "81": 10.01083, - "82": 9.94172, - "83": 9.9206, - "84": 9.85049, - "85": 9.84781, - "86": 9.93316, - "87": 9.96436, - "88": 9.93256, - "89": 9.87234, - "90": 9.87713, - "91": 9.89316, - "92": 9.87578, - "93": 9.81095, - "94": 9.89088, - "95": 9.87267, - "96": 9.85399, - "97": 9.79962, - "98": 9.83397, - "99": 9.87153, - "100": 9.76507 + "73": 10.02503, + "74": 10.00651, + "75": 9.98677, + "76": 10.00597, + "77": 10.00709, + "78": 9.95735, + "79": 9.96947, + "80": 9.98336, + "81": 10.01128, + "82": 9.94198, + "83": 9.9208, + "84": 9.85035, + "85": 9.84797, + "86": 9.93302, + "87": 9.96425, + "88": 9.93278, + "89": 9.87206, + "90": 9.87717, + "91": 9.89303, + "92": 9.87591, + "93": 9.81089, + "94": 9.89098, + "95": 9.87301, + "96": 9.854, + "97": 9.79985, + "98": 9.83386, + "99": 9.87162, + "100": 9.76501 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36422.0, - "2": 36679.0, - "3": 36905.0, - "4": 36533.0, - "5": 36340.0, - "6": 36090.0, - "7": 36769.0, - "8": 36136.0, - "9": 37073.0, - "10": 37549.0, - "11": 36644.0, - "12": 35731.0, - "13": 36450.0, - "14": 36585.0, - "15": 35800.0, - "16": 35805.0, - "17": 36233.0, - "18": 36358.0, - "19": 36139.0, - "20": 36017.0, - "21": 36415.0, - "22": 36481.0, - "23": 36460.0, - "24": 36548.0, - "25": 35475.0, - "26": 36208.0, - "27": 36177.0, - "28": 35154.0, - "29": 35832.0, - "30": 35463.0, - "31": 36090.0, - "32": 36563.0, - "33": 35914.0, - "34": 36016.0, - "35": 36309.0, - "36": 35396.0, - "37": 36540.0, - "38": 35916.0, - "39": 36651.0, - "40": 37650.0, - "41": 35665.0, - "42": 35649.0, - "43": 37153.0, - "44": 36787.0, - "45": 39375.0, - "46": 38135.0, - "47": 38693.0, - "48": 39208.0, - "49": 42324.0, - "50": 39548.0, - "51": 38405.0, - "52": 41739.0, - "53": 40631.0, - "54": 41482.0, - "55": 39474.0, - "56": 41606.0, - "57": 37121.0, - "58": 46258.0, - "59": 42095.0, - "60": 43008.0, - "61": 42348.0, - "62": 45250.0, - "63": 44233.0, - "64": 49282.0, - "65": 42343.0, - "66": 46049.0, - "67": 49857.0, - "68": 46130.0, - "69": 46746.0, - "70": 46986.0, - "71": 46617.0, - "72": 45684.0, - "73": 47285.0, - "74": 46144.0, - "75": 48939.0, - "76": 47734.0, - "77": 48245.0, - "78": 48654.0, - "79": 48107.0, - "80": 45827.0, - "81": 54970.0, - "82": 41870.0, - "83": 46666.0, - "84": 45356.0, - "85": 45792.0, - "86": 45906.0, - "87": 36357.0, - "88": 45646.0, - "89": 48764.0, - "90": 56944.0, - "91": 39633.0, - "92": 49351.0, - "93": 44601.0, - "94": 42318.0, - "95": 45069.0, - "96": 50508.0, - "97": 46117.0, - "98": 50582.0, - "99": 45303.0, - "100": 46607.0 + "1": 36227.0, + "2": 37201.0, + "3": 36774.0, + "4": 36570.0, + "5": 36726.0, + "6": 35453.0, + "7": 36952.0, + "8": 36230.0, + "9": 37164.0, + "10": 37327.0, + "11": 36787.0, + "12": 35527.0, + "13": 36236.0, + "14": 36776.0, + "15": 35521.0, + "16": 36024.0, + "17": 36158.0, + "18": 36460.0, + "19": 36280.0, + "20": 36245.0, + "21": 36531.0, + "22": 36435.0, + "23": 36450.0, + "24": 37114.0, + "25": 35514.0, + "26": 36289.0, + "27": 36220.0, + "28": 35094.0, + "29": 35685.0, + "30": 35235.0, + "31": 36128.0, + "32": 36397.0, + "33": 35851.0, + "34": 35612.0, + "35": 36610.0, + "36": 35768.0, + "37": 36402.0, + "38": 35569.0, + "39": 36310.0, + "40": 37593.0, + "41": 36111.0, + "42": 35320.0, + "43": 37666.0, + "44": 36410.0, + "45": 39411.0, + "46": 38337.0, + "47": 38141.0, + "48": 39163.0, + "49": 42791.0, + "50": 39350.0, + "51": 38770.0, + "52": 41454.0, + "53": 40064.0, + "54": 41412.0, + "55": 39457.0, + "56": 41555.0, + "57": 37183.0, + "58": 46390.0, + "59": 42779.0, + "60": 42192.0, + "61": 42281.0, + "62": 45440.0, + "63": 44644.0, + "64": 49137.0, + "65": 41769.0, + "66": 45654.0, + "67": 49704.0, + "68": 46553.0, + "69": 46016.0, + "70": 46937.0, + "71": 46350.0, + "72": 45770.0, + "73": 47491.0, + "74": 46137.0, + "75": 48941.0, + "76": 48261.0, + "77": 47890.0, + "78": 48949.0, + "79": 48015.0, + "80": 45941.0, + "81": 55431.0, + "82": 41811.0, + "83": 46772.0, + "84": 45180.0, + "85": 45564.0, + "86": 46154.0, + "87": 36767.0, + "88": 45300.0, + "89": 48760.0, + "90": 57596.0, + "91": 39387.0, + "92": 49929.0, + "93": 45177.0, + "94": 42214.0, + "95": 45091.0, + "96": 50439.0, + "97": 46128.0, + "98": 50578.0, + "99": 45725.0, + "100": 47131.0 } }, "mem-allocated-bytes": { @@ -219,105 +219,105 @@ "step_interval": 1, "values": { "1": 892868608.0, - "2": 892868096.0, - "3": 892870144.0, + "2": 892867584.0, + "3": 892869120.0, "4": 892869632.0, - "5": 892867584.0, - "6": 892865536.0, - "7": 892869120.0, - "8": 892869120.0, + "5": 892868096.0, + "6": 892866048.0, + "7": 892868608.0, + "8": 892870144.0, "9": 892871680.0, "10": 892871168.0, - "11": 892869120.0, - "12": 892870656.0, + "11": 892869632.0, + "12": 892870144.0, "13": 892869632.0, "14": 892871168.0, - "15": 892870144.0, - "16": 892871680.0, - "17": 892870144.0, - "18": 892871168.0, + "15": 892870656.0, + "16": 892870144.0, + "17": 892870656.0, + "18": 892870656.0, "19": 892871680.0, - "20": 892870656.0, - "21": 892869120.0, - "22": 892865536.0, - "23": 892869120.0, - "24": 892870656.0, - "25": 892863488.0, - "26": 892871168.0, - "27": 892869632.0, - "28": 892868608.0, + "20": 892870144.0, + "21": 892867584.0, + "22": 892866560.0, + "23": 892869632.0, + "24": 892869120.0, + "25": 892864512.0, + "26": 892870144.0, + "27": 892870144.0, + "28": 892865536.0, "29": 892870144.0, - "30": 892871168.0, - "31": 892870656.0, - "32": 892867072.0, - "33": 892871680.0, + "30": 892870144.0, + "31": 892871680.0, + "32": 892866048.0, + "33": 892870656.0, "34": 892868608.0, - "35": 892868096.0, - "36": 892869120.0, - "37": 892869120.0, - "38": 892868608.0, - "39": 892867584.0, - "40": 892867584.0, - "41": 892871168.0, - "42": 892868608.0, + "35": 892869120.0, + "36": 892868608.0, + "37": 892866048.0, + "38": 892869120.0, + "39": 892866048.0, + "40": 892869120.0, + "41": 892869632.0, + "42": 892866560.0, "43": 892869120.0, - "44": 892868096.0, - "45": 892869632.0, - "46": 892868608.0, - "47": 892870144.0, - "48": 892868608.0, + "44": 892867584.0, + "45": 892870144.0, + "46": 892869120.0, + "47": 892869632.0, + "48": 892869632.0, "49": 892870656.0, "50": 892868096.0, - "51": 892869632.0, + "51": 892870144.0, "52": 892870144.0, "53": 892868608.0, - "54": 892867072.0, + "54": 892867584.0, "55": 892868096.0, - "56": 892866560.0, + "56": 892867072.0, "57": 892869120.0, "58": 892871168.0, "59": 892868096.0, "60": 892866560.0, "61": 892865536.0, - "62": 892868608.0, - "63": 892869632.0, - "64": 892872704.0, - "65": 892868608.0, - "66": 892868608.0, - "67": 892868608.0, - "68": 892867072.0, - "69": 892868608.0, - "70": 892866560.0, + "62": 892868096.0, + "63": 892868096.0, + "64": 892872192.0, + "65": 892866560.0, + "66": 892868096.0, + "67": 892869120.0, + "68": 892868096.0, + "69": 892868096.0, + "70": 892866048.0, "71": 892868608.0, "72": 892868096.0, - "73": 892867072.0, - "74": 892866560.0, - "75": 892865536.0, - "76": 892870656.0, - "77": 892867072.0, - "78": 892867584.0, - "79": 892868608.0, + "73": 892868096.0, + "74": 892866048.0, + "75": 892866048.0, + "76": 892870144.0, + "77": 892868608.0, + "78": 892865536.0, + "79": 892870144.0, "80": 892868608.0, - "81": 892866048.0, - "82": 892870144.0, - "83": 892866048.0, + "81": 892866560.0, + "82": 892868608.0, + "83": 892866560.0, "84": 892870656.0, - "85": 892866048.0, - "86": 892868608.0, - "87": 892866560.0, - "88": 892867072.0, - "89": 892870144.0, + "85": 892865536.0, + "86": 892868096.0, + "87": 892867584.0, + "88": 892866048.0, + "89": 892868608.0, "90": 892871168.0, "91": 892866560.0, - "92": 892869120.0, - "93": 892867072.0, + "92": 892870656.0, + "93": 892866560.0, "94": 892866560.0, - "95": 892875776.0, + "95": 892876288.0, "96": 892868096.0, "97": 892870144.0, - "98": 892871168.0, - "99": 892869632.0, - "100": 892875264.0 + "98": 892872704.0, + "99": 892870656.0, + "100": 892876800.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1943236608.0, - "2": 2184390656.0, - "3": 2184390656.0, - "4": 2186935808.0, - "5": 2186935808.0, - "6": 2186935808.0, - "7": 2186935808.0, - "8": 2186935808.0, - "9": 2186935808.0, - "10": 2186935808.0, - "11": 2186935808.0, - "12": 2186935808.0, - "13": 2186935808.0, - "14": 2186935808.0, - "15": 2186935808.0, - "16": 2186935808.0, - "17": 2186935808.0, - "18": 2186935808.0, - "19": 2186935808.0, - "20": 2186935808.0, - "21": 2186935808.0, - "22": 2186935808.0, - "23": 2186935808.0, - "24": 2186935808.0, - "25": 2186935808.0, - "26": 2186935808.0, - "27": 2186935808.0, - "28": 2186935808.0, - "29": 2186935808.0, - "30": 2186935808.0, - "31": 2186935808.0, - "32": 2186935808.0, - "33": 2186935808.0, - "34": 2186935808.0, - "35": 2186935808.0, - "36": 2186935808.0, - "37": 2186935808.0, - "38": 2186935808.0, - "39": 2186935808.0, - "40": 2186935808.0, - "41": 2186935808.0, - "42": 2186935808.0, - "43": 2186935808.0, - "44": 2186935808.0, - "45": 2186935808.0, - "46": 2186935808.0, - "47": 2186935808.0, - "48": 2186935808.0, - "49": 2186935808.0, - "50": 2186935808.0, - "51": 2186935808.0, - "52": 2186935808.0, - "53": 2186935808.0, - "54": 2186935808.0, - "55": 2186935808.0, - "56": 2186935808.0, - "57": 2186935808.0, - "58": 2186935808.0, - "59": 2186935808.0, - "60": 2186935808.0, - "61": 2186935808.0, - "62": 2186935808.0, - "63": 2186935808.0, - "64": 2186935808.0, - "65": 2186935808.0, - "66": 2186935808.0, - "67": 2186935808.0, - "68": 2186935808.0, - "69": 2186935808.0, - "70": 2186935808.0, - "71": 2186935808.0, - "72": 2186935808.0, - "73": 2186935808.0, - "74": 2186935808.0, - "75": 2186935808.0, - "76": 2186935808.0, - "77": 2186935808.0, - "78": 2186935808.0, - "79": 2186935808.0, - "80": 2186935808.0, - "81": 2186935808.0, - "82": 2186935808.0, - "83": 2186935808.0, - "84": 2186935808.0, - "85": 2186935808.0, - "86": 2186935808.0, - "87": 2186935808.0, - "88": 2186935808.0, - "89": 2186935808.0, - "90": 2186935808.0, - "91": 2186935808.0, - "92": 2186935808.0, - "93": 2186935808.0, - "94": 2186935808.0, - "95": 2188984320.0, - "96": 2188984320.0, - "97": 2188984320.0, - "98": 2188984320.0, - "99": 2188984320.0, - "100": 2192725504.0 + "1": 1942771712.0, + "2": 2185236480.0, + "3": 2185236480.0, + "4": 2185236480.0, + "5": 2185236480.0, + "6": 2185236480.0, + "7": 2185236480.0, + "8": 2185236480.0, + "9": 2185236480.0, + "10": 2185236480.0, + "11": 2185236480.0, + "12": 2185236480.0, + "13": 2185236480.0, + "14": 2185236480.0, + "15": 2185236480.0, + "16": 2185236480.0, + "17": 2185236480.0, + "18": 2185236480.0, + "19": 2185236480.0, + "20": 2185236480.0, + "21": 2185236480.0, + "22": 2185236480.0, + "23": 2185236480.0, + "24": 2185236480.0, + "25": 2185236480.0, + "26": 2185236480.0, + "27": 2185236480.0, + "28": 2185236480.0, + "29": 2185236480.0, + "30": 2185236480.0, + "31": 2185236480.0, + "32": 2185236480.0, + "33": 2185236480.0, + "34": 2185236480.0, + "35": 2185236480.0, + "36": 2185236480.0, + "37": 2185236480.0, + "38": 2185236480.0, + "39": 2185236480.0, + "40": 2185236480.0, + "41": 2185236480.0, + "42": 2185236480.0, + "43": 2185236480.0, + "44": 2185236480.0, + "45": 2185236480.0, + "46": 2185236480.0, + "47": 2185236480.0, + "48": 2185236480.0, + "49": 2185236480.0, + "50": 2185236480.0, + "51": 2185236480.0, + "52": 2185236480.0, + "53": 2185236480.0, + "54": 2185236480.0, + "55": 2185236480.0, + "56": 2185236480.0, + "57": 2185236480.0, + "58": 2185236480.0, + "59": 2185236480.0, + "60": 2185236480.0, + "61": 2185236480.0, + "62": 2185236480.0, + "63": 2185236480.0, + "64": 2185236480.0, + "65": 2185236480.0, + "66": 2185236480.0, + "67": 2185236480.0, + "68": 2185236480.0, + "69": 2185236480.0, + "70": 2185236480.0, + "71": 2185236480.0, + "72": 2185236480.0, + "73": 2185236480.0, + "74": 2185236480.0, + "75": 2185236480.0, + "76": 2185236480.0, + "77": 2185236480.0, + "78": 2185236480.0, + "79": 2185236480.0, + "80": 2185236480.0, + "81": 2185236480.0, + "82": 2185236480.0, + "83": 2185236480.0, + "84": 2185236480.0, + "85": 2185236480.0, + "86": 2185236480.0, + "87": 2185236480.0, + "88": 2185236480.0, + "89": 2185236480.0, + "90": 2185236480.0, + "91": 2185236480.0, + "92": 2185236480.0, + "93": 2185236480.0, + "94": 2185236480.0, + "95": 2190002688.0, + "96": 2190002688.0, + "97": 2190002688.0, + "98": 2190002688.0, + "99": 2190002688.0, + "100": 2191467520.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.6397, - "3": 0.39365, - "4": 0.37927, - "5": 0.3691, - "6": 0.37122, - "7": 0.36778, - "8": 0.36883, - "9": 0.36404, - "10": 0.36715, - "11": 0.3637, - "12": 0.37218, - "13": 0.36619, - "14": 0.371, - "15": 0.37304, - "16": 0.37059, - "17": 0.36162, - "18": 0.3628, - "19": 0.36531, - "20": 0.36518, - "21": 0.36185, - "22": 0.36187, - "23": 0.36053, - "24": 0.36358, - "25": 0.36142, - "26": 0.36186, - "27": 0.35994, - "28": 0.3627, - "29": 0.36292, - "30": 0.36501, - "31": 0.36497, - "32": 0.3633, - "33": 0.36591, - "34": 0.36891, - "35": 0.36952, - "36": 0.37369, - "37": 0.36547, - "38": 0.36624, - "39": 0.36781, - "40": 0.36303, - "41": 0.36236, - "42": 0.36798, - "43": 0.36605, - "44": 0.36334, - "45": 0.3697, - "46": 0.36169, - "47": 0.36368, - "48": 0.36178, - "49": 0.36626, - "50": 0.36768, - "51": 0.50792, - "52": 0.39563, - "53": 0.42675, - "54": 0.36322, - "55": 0.37513, - "56": 0.36126, - "57": 0.36372, - "58": 0.36697, - "59": 0.36853, - "60": 0.36217, - "61": 0.36309, - "62": 0.36113, - "63": 0.36547, - "64": 0.3627, - "65": 0.36582, - "66": 0.36369, - "67": 0.36675, - "68": 0.36503, - "69": 0.371, - "70": 0.36509, - "71": 0.36434, - "72": 0.366, - "73": 0.36486, - "74": 0.36671, - "75": 0.36875, - "76": 0.3649, - "77": 0.36961, - "78": 0.36617, - "79": 0.36914, - "80": 0.36979, - "81": 0.37089, - "82": 0.36879, - "83": 0.37107, - "84": 0.37124, - "85": 0.54996, - "86": 0.3798, - "87": 0.3643, - "88": 0.37963, - "89": 0.3752, - "90": 0.37278, - "91": 0.36951, - "92": 0.3846, - "93": 0.37218, - "94": 0.37844, - "95": 0.38989, - "96": 0.39169, - "97": 0.39722, - "98": 0.3892, - "99": 0.38178, - "100": 0.38838 + "2": 8.3943, + "3": 0.35679, + "4": 0.31335, + "5": 0.31492, + "6": 0.31672, + "7": 0.31181, + "8": 0.31416, + "9": 0.31079, + "10": 0.31425, + "11": 0.31258, + "12": 0.32064, + "13": 0.3113, + "14": 0.30822, + "15": 0.3094, + "16": 0.30839, + "17": 0.30992, + "18": 0.30931, + "19": 0.30793, + "20": 0.31001, + "21": 0.30342, + "22": 0.30529, + "23": 0.30531, + "24": 0.30356, + "25": 0.30675, + "26": 0.30814, + "27": 0.30568, + "28": 0.30511, + "29": 0.30924, + "30": 0.30828, + "31": 0.31135, + "32": 0.3093, + "33": 0.30968, + "34": 0.31158, + "35": 0.30945, + "36": 0.30729, + "37": 0.30748, + "38": 0.30777, + "39": 0.30933, + "40": 0.30804, + "41": 0.30755, + "42": 0.30625, + "43": 0.30736, + "44": 0.31048, + "45": 0.30798, + "46": 0.30525, + "47": 0.30446, + "48": 0.30903, + "49": 0.30856, + "50": 0.30479, + "51": 0.46129, + "52": 2.65987, + "53": 0.31114, + "54": 0.30485, + "55": 0.30928, + "56": 0.31108, + "57": 0.30846, + "58": 0.30707, + "59": 0.30645, + "60": 0.30617, + "61": 0.30655, + "62": 0.30785, + "63": 0.30733, + "64": 0.30579, + "65": 0.30717, + "66": 0.30996, + "67": 0.30693, + "68": 0.30695, + "69": 0.30959, + "70": 0.3054, + "71": 0.31011, + "72": 0.30617, + "73": 0.30583, + "74": 0.30501, + "75": 0.30502, + "76": 0.30693, + "77": 0.31003, + "78": 0.30536, + "79": 0.30628, + "80": 0.31141, + "81": 0.30471, + "82": 0.30679, + "83": 0.30966, + "84": 0.31271, + "85": 0.31752, + "86": 0.31595, + "87": 0.30556, + "88": 0.30899, + "89": 0.31041, + "90": 0.30808, + "91": 0.31775, + "92": 0.32248, + "93": 0.32439, + "94": 0.30868, + "95": 0.32654, + "96": 0.328, + "97": 0.32584, + "98": 0.32658, + "99": 0.32031, + "100": 0.34272 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json index d4eaa62dbaf..ad2d31db316 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92624, - "2": 10.91761, - "3": 10.92333, - "4": 10.92688, - "5": 10.92998, - "6": 10.92516, + "1": 10.92561, + "2": 10.91856, + "3": 10.92257, + "4": 10.92726, + "5": 10.92964, + "6": 10.92362, "7": 10.9275, - "8": 10.91969, - "9": 10.92241, - "10": 10.92325, - "11": 10.9076, - "12": 10.92305, - "13": 10.9061, - "14": 10.88901, - "15": 10.89569, - "16": 10.87458, - "17": 10.88061, - "18": 10.87989, - "19": 10.87749, - "20": 10.82246, - "21": 10.82186, - "22": 10.7974, - "23": 10.80849, - "24": 10.77057, - "25": 10.77862, - "26": 10.75668, - "27": 10.74862, - "28": 10.68541, - "29": 10.65859, - "30": 10.62605, - "31": 10.61208, - "32": 10.60629, - "33": 10.57851, - "34": 10.53548, + "8": 10.91971, + "9": 10.92189, + "10": 10.92381, + "11": 10.90772, + "12": 10.92237, + "13": 10.907, + "14": 10.88924, + "15": 10.89606, + "16": 10.87509, + "17": 10.88069, + "18": 10.87988, + "19": 10.8779, + "20": 10.82287, + "21": 10.82214, + "22": 10.79726, + "23": 10.80932, + "24": 10.77051, + "25": 10.77867, + "26": 10.75636, + "27": 10.74899, + "28": 10.68475, + "29": 10.65911, + "30": 10.6272, + "31": 10.61296, + "32": 10.6061, + "33": 10.57894, + "34": 10.536, "35": 10.53752, - "36": 10.52815, - "37": 10.49657, - "38": 10.49494, - "39": 10.46031, - "40": 10.44224, - "41": 10.42194, - "42": 10.41512, - "43": 10.39777, - "44": 10.36628, - "45": 10.37246, - "46": 10.33638, - "47": 10.31798, - "48": 10.28671, - "49": 10.27533, - "50": 10.27489, - "51": 10.27015, - "52": 10.22169, - "53": 10.22939, - "54": 10.19442, - "55": 10.17402, - "56": 10.18836, - "57": 10.17883, - "58": 10.18533, - "59": 10.13453, - "60": 10.15581, - "61": 10.10616, - "62": 10.074, - "63": 10.13216, - "64": 10.09238, - "65": 10.06787, - "66": 10.09506, - "67": 10.07264, - "68": 10.03606, - "69": 10.05538, - "70": 10.03315, - "71": 10.04934, - "72": 10.04115, - "73": 10.03896, - "74": 10.0216, + "36": 10.52821, + "37": 10.4971, + "38": 10.49446, + "39": 10.46075, + "40": 10.44199, + "41": 10.42128, + "42": 10.41473, + "43": 10.39834, + "44": 10.36684, + "45": 10.37233, + "46": 10.33579, + "47": 10.31804, + "48": 10.28691, + "49": 10.27523, + "50": 10.27494, + "51": 10.26955, + "52": 10.22127, + "53": 10.22941, + "54": 10.19422, + "55": 10.17429, + "56": 10.18853, + "57": 10.17897, + "58": 10.18526, + "59": 10.13483, + "60": 10.15585, + "61": 10.10615, + "62": 10.07405, + "63": 10.13177, + "64": 10.09204, + "65": 10.06791, + "66": 10.095, + "67": 10.07294, + "68": 10.03607, + "69": 10.05491, + "70": 10.03321, + "71": 10.04987, + "72": 10.04091, + "73": 10.03878, + "74": 10.02114, "75": 9.9823, - "76": 10.00321, - "77": 10.0147, - "78": 9.96885, - "79": 9.96967, - "80": 9.98518, - "81": 10.01118, - "82": 9.94219, - "83": 9.91692, - "84": 9.8533, - "85": 9.83713, - "86": 9.94372, - "87": 9.95587, - "88": 9.93524, - "89": 9.87174, - "90": 9.87886, - "91": 9.89547, - "92": 9.86943, - "93": 9.80146, - "94": 9.88892, - "95": 9.86048, - "96": 9.84941, - "97": 9.79913, - "98": 9.83243, - "99": 9.8647, - "100": 9.75978 + "76": 10.00354, + "77": 10.01483, + "78": 9.96817, + "79": 9.96957, + "80": 9.98504, + "81": 10.01108, + "82": 9.9421, + "83": 9.91677, + "84": 9.85306, + "85": 9.83728, + "86": 9.94391, + "87": 9.95597, + "88": 9.93542, + "89": 9.87129, + "90": 9.87899, + "91": 9.89527, + "92": 9.86926, + "93": 9.80185, + "94": 9.88937, + "95": 9.86027, + "96": 9.84922, + "97": 9.79903, + "98": 9.83241, + "99": 9.86401, + "100": 9.76018 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36362.0, - "2": 36609.0, - "3": 36699.0, - "4": 36555.0, - "5": 36453.0, - "6": 36089.0, - "7": 37058.0, - "8": 36230.0, - "9": 37158.0, - "10": 37488.0, - "11": 36351.0, - "12": 35467.0, - "13": 36613.0, - "14": 36715.0, - "15": 36545.0, - "16": 35601.0, - "17": 36415.0, - "18": 36855.0, - "19": 36832.0, - "20": 36630.0, - "21": 35929.0, - "22": 36615.0, - "23": 36863.0, - "24": 36540.0, - "25": 36141.0, - "26": 36330.0, - "27": 36011.0, - "28": 35481.0, - "29": 35864.0, - "30": 35285.0, - "31": 36267.0, - "32": 36439.0, - "33": 35284.0, - "34": 35815.0, - "35": 36651.0, - "36": 35025.0, - "37": 36022.0, - "38": 35730.0, - "39": 36407.0, - "40": 37543.0, - "41": 36402.0, - "42": 35346.0, - "43": 37029.0, - "44": 36347.0, - "45": 39301.0, - "46": 38134.0, - "47": 38278.0, - "48": 38458.0, - "49": 42002.0, - "50": 38854.0, - "51": 38657.0, - "52": 41903.0, - "53": 40194.0, - "54": 41670.0, - "55": 38786.0, - "56": 40937.0, - "57": 36754.0, - "58": 45577.0, - "59": 42100.0, - "60": 42166.0, - "61": 41783.0, - "62": 44364.0, - "63": 43830.0, - "64": 50186.0, - "65": 41853.0, - "66": 45150.0, - "67": 50276.0, - "68": 47034.0, - "69": 45199.0, - "70": 47305.0, - "71": 45654.0, - "72": 44705.0, - "73": 47912.0, - "74": 45912.0, - "75": 47295.0, - "76": 46874.0, - "77": 47833.0, - "78": 49256.0, - "79": 47375.0, - "80": 45364.0, - "81": 53599.0, - "82": 42559.0, - "83": 46334.0, - "84": 45573.0, - "85": 45415.0, - "86": 45108.0, - "87": 36933.0, - "88": 46121.0, - "89": 49063.0, - "90": 56728.0, - "91": 38149.0, - "92": 49957.0, - "93": 46429.0, - "94": 43030.0, - "95": 45818.0, - "96": 50189.0, - "97": 47382.0, - "98": 49426.0, - "99": 43744.0, - "100": 45484.0 + "1": 36436.0, + "2": 36501.0, + "3": 36802.0, + "4": 36766.0, + "5": 36386.0, + "6": 35733.0, + "7": 37148.0, + "8": 36168.0, + "9": 37700.0, + "10": 37692.0, + "11": 36380.0, + "12": 35362.0, + "13": 36444.0, + "14": 37198.0, + "15": 36113.0, + "16": 36080.0, + "17": 35890.0, + "18": 36677.0, + "19": 36875.0, + "20": 36654.0, + "21": 35934.0, + "22": 36630.0, + "23": 36555.0, + "24": 36447.0, + "25": 35789.0, + "26": 36432.0, + "27": 36480.0, + "28": 35574.0, + "29": 35892.0, + "30": 35261.0, + "31": 36004.0, + "32": 36774.0, + "33": 35429.0, + "34": 35421.0, + "35": 36348.0, + "36": 35172.0, + "37": 36118.0, + "38": 35314.0, + "39": 36245.0, + "40": 37579.0, + "41": 35853.0, + "42": 35642.0, + "43": 36534.0, + "44": 36152.0, + "45": 39483.0, + "46": 38103.0, + "47": 37913.0, + "48": 38654.0, + "49": 42357.0, + "50": 38920.0, + "51": 38654.0, + "52": 41728.0, + "53": 40126.0, + "54": 41345.0, + "55": 38968.0, + "56": 40444.0, + "57": 36849.0, + "58": 45895.0, + "59": 41886.0, + "60": 42200.0, + "61": 41553.0, + "62": 44568.0, + "63": 43769.0, + "64": 49912.0, + "65": 41452.0, + "66": 45193.0, + "67": 49905.0, + "68": 47291.0, + "69": 45132.0, + "70": 47912.0, + "71": 45345.0, + "72": 45252.0, + "73": 47856.0, + "74": 46077.0, + "75": 46976.0, + "76": 47228.0, + "77": 47619.0, + "78": 48947.0, + "79": 47363.0, + "80": 45590.0, + "81": 53548.0, + "82": 42539.0, + "83": 46174.0, + "84": 46192.0, + "85": 45427.0, + "86": 45044.0, + "87": 37111.0, + "88": 46323.0, + "89": 49563.0, + "90": 56927.0, + "91": 38107.0, + "92": 49565.0, + "93": 46649.0, + "94": 42544.0, + "95": 45963.0, + "96": 49752.0, + "97": 47239.0, + "98": 49652.0, + "99": 44234.0, + "100": 45362.0 } }, "mem-allocated-bytes": { @@ -218,49 +218,49 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 892871168.0, - "2": 892866048.0, - "3": 892867584.0, - "4": 892867584.0, + "1": 892870144.0, + "2": 892867584.0, + "3": 892866560.0, + "4": 892868096.0, "5": 892868096.0, - "6": 892871168.0, - "7": 892869632.0, - "8": 892867072.0, - "9": 892868096.0, + "6": 892871680.0, + "7": 892870144.0, + "8": 892866560.0, + "9": 892867072.0, "10": 892866560.0, - "11": 892868096.0, - "12": 892865536.0, - "13": 892869632.0, - "14": 892869632.0, - "15": 892868096.0, - "16": 892866560.0, - "17": 892867584.0, - "18": 892866560.0, - "19": 892872704.0, - "20": 892866048.0, - "21": 892867072.0, - "22": 892868096.0, + "11": 892869120.0, + "12": 892867072.0, + "13": 892869120.0, + "14": 892867584.0, + "15": 892868608.0, + "16": 892867072.0, + "17": 892866560.0, + "18": 892867584.0, + "19": 892871168.0, + "20": 892867072.0, + "21": 892866560.0, + "22": 892867584.0, "23": 892868608.0, "24": 892868096.0, - "25": 892866560.0, + "25": 892868096.0, "26": 892864512.0, - "27": 892867584.0, - "28": 892867584.0, - "29": 892871168.0, - "30": 892867072.0, - "31": 892868096.0, - "32": 892865536.0, - "33": 892868096.0, - "34": 892866048.0, - "35": 892866560.0, - "36": 892868608.0, - "37": 892869632.0, - "38": 892866560.0, + "27": 892869120.0, + "28": 892868608.0, + "29": 892869632.0, + "30": 892866560.0, + "31": 892867072.0, + "32": 892866048.0, + "33": 892868608.0, + "34": 892867584.0, + "35": 892866048.0, + "36": 892868096.0, + "37": 892870144.0, + "38": 892867072.0, "39": 892867584.0, - "40": 892869632.0, - "41": 892869120.0, - "42": 892867072.0, - "43": 892865536.0, + "40": 892870656.0, + "41": 892868096.0, + "42": 892868096.0, + "43": 892864512.0, "44": 892867584.0, "45": 892868096.0, "46": 892868608.0, @@ -268,56 +268,56 @@ "48": 892866048.0, "49": 892866048.0, "50": 892869632.0, - "51": 892865024.0, - "52": 892868096.0, + "51": 892866048.0, + "52": 892867072.0, "53": 892866560.0, - "54": 892865536.0, - "55": 892868608.0, - "56": 892866048.0, - "57": 892868096.0, - "58": 892868608.0, - "59": 892866560.0, - "60": 892865536.0, - "61": 892865024.0, + "54": 892867072.0, + "55": 892869120.0, + "56": 892867584.0, + "57": 892868608.0, + "58": 892868096.0, + "59": 892867072.0, + "60": 892865024.0, + "61": 892865536.0, "62": 892866048.0, - "63": 892865024.0, - "64": 892867584.0, - "65": 892866048.0, - "66": 892868096.0, - "67": 892865536.0, - "68": 892868608.0, + "63": 892865536.0, + "64": 892868096.0, + "65": 892867072.0, + "66": 892870144.0, + "67": 892866048.0, + "68": 892869120.0, "69": 892866048.0, - "70": 892864512.0, - "71": 892866560.0, + "70": 892866048.0, + "71": 892867584.0, "72": 892870144.0, - "73": 892866048.0, - "74": 892866048.0, + "73": 892865536.0, + "74": 892865024.0, "75": 892868608.0, - "76": 892866560.0, - "77": 892861440.0, - "78": 892866560.0, - "79": 892867072.0, + "76": 892868096.0, + "77": 892862464.0, + "78": 892864512.0, + "79": 892865536.0, "80": 892865024.0, "81": 892861952.0, - "82": 892862464.0, + "82": 892862976.0, "83": 892867072.0, - "84": 892861440.0, - "85": 892859392.0, - "86": 892853248.0, - "87": 892865536.0, - "88": 892869120.0, - "89": 892857856.0, - "90": 892859904.0, - "91": 892866560.0, - "92": 892855808.0, - "93": 892861440.0, + "84": 892863488.0, + "85": 892859904.0, + "86": 892852736.0, + "87": 892865024.0, + "88": 892868096.0, + "89": 892858368.0, + "90": 892859392.0, + "91": 892866048.0, + "92": 892854272.0, + "93": 892861952.0, "94": 892855808.0, "95": 892853248.0, - "96": 892867584.0, - "97": 892853760.0, - "98": 892847104.0, - "99": 892852736.0, - "100": 892852224.0 + "96": 892868096.0, + "97": 892853248.0, + "98": 892846592.0, + "99": 892853760.0, + "100": 892851712.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1921131008.0, - "2": 2156760064.0, - "3": 2156760064.0, - "4": 2156760064.0, - "5": 2156760064.0, - "6": 2159165952.0, - "7": 2159165952.0, - "8": 2159165952.0, - "9": 2159165952.0, - "10": 2159165952.0, - "11": 2159165952.0, - "12": 2159165952.0, - "13": 2159165952.0, - "14": 2159165952.0, - "15": 2159165952.0, - "16": 2159165952.0, - "17": 2159165952.0, - "18": 2159165952.0, - "19": 2159417344.0, - "20": 2159417344.0, - "21": 2159417344.0, - "22": 2159417344.0, - "23": 2159417344.0, - "24": 2159417344.0, - "25": 2159417344.0, - "26": 2159417344.0, - "27": 2159417344.0, - "28": 2159417344.0, - "29": 2159417344.0, - "30": 2159417344.0, - "31": 2159417344.0, - "32": 2159417344.0, - "33": 2159417344.0, - "34": 2159417344.0, - "35": 2159417344.0, - "36": 2159417344.0, - "37": 2159417344.0, - "38": 2159417344.0, - "39": 2159417344.0, - "40": 2159417344.0, - "41": 2159417344.0, - "42": 2159417344.0, - "43": 2159417344.0, - "44": 2159417344.0, - "45": 2159417344.0, - "46": 2159417344.0, - "47": 2159417344.0, - "48": 2159417344.0, - "49": 2159417344.0, - "50": 2159417344.0, - "51": 2159417344.0, - "52": 2159417344.0, - "53": 2159417344.0, - "54": 2159417344.0, - "55": 2159417344.0, - "56": 2159417344.0, - "57": 2159417344.0, - "58": 2159417344.0, - "59": 2159417344.0, - "60": 2159417344.0, - "61": 2159417344.0, - "62": 2159417344.0, - "63": 2159417344.0, - "64": 2159417344.0, - "65": 2159417344.0, - "66": 2159417344.0, - "67": 2159417344.0, - "68": 2159417344.0, - "69": 2159417344.0, - "70": 2159417344.0, - "71": 2159417344.0, - "72": 2159417344.0, - "73": 2159417344.0, - "74": 2159417344.0, - "75": 2159417344.0, - "76": 2159417344.0, - "77": 2159417344.0, - "78": 2159417344.0, - "79": 2159417344.0, - "80": 2159417344.0, - "81": 2159417344.0, - "82": 2159417344.0, - "83": 2159417344.0, - "84": 2159417344.0, - "85": 2159417344.0, - "86": 2159417344.0, - "87": 2159417344.0, - "88": 2159417344.0, - "89": 2159417344.0, - "90": 2159417344.0, - "91": 2159417344.0, - "92": 2159417344.0, - "93": 2159417344.0, - "94": 2159417344.0, - "95": 2159417344.0, - "96": 2159417344.0, - "97": 2159417344.0, - "98": 2159417344.0, - "99": 2159417344.0, - "100": 2159417344.0 + "1": 1921322496.0, + "2": 2156503040.0, + "3": 2156503040.0, + "4": 2156758528.0, + "5": 2156758528.0, + "6": 2159042560.0, + "7": 2159042560.0, + "8": 2159042560.0, + "9": 2159042560.0, + "10": 2159042560.0, + "11": 2159042560.0, + "12": 2159042560.0, + "13": 2159042560.0, + "14": 2159042560.0, + "15": 2159042560.0, + "16": 2159042560.0, + "17": 2159042560.0, + "18": 2159042560.0, + "19": 2159231488.0, + "20": 2159231488.0, + "21": 2159231488.0, + "22": 2159231488.0, + "23": 2159231488.0, + "24": 2159231488.0, + "25": 2159231488.0, + "26": 2159231488.0, + "27": 2159231488.0, + "28": 2159231488.0, + "29": 2159231488.0, + "30": 2159231488.0, + "31": 2159231488.0, + "32": 2159231488.0, + "33": 2159231488.0, + "34": 2159231488.0, + "35": 2159231488.0, + "36": 2159231488.0, + "37": 2159231488.0, + "38": 2159231488.0, + "39": 2159231488.0, + "40": 2159231488.0, + "41": 2159231488.0, + "42": 2159231488.0, + "43": 2159231488.0, + "44": 2159231488.0, + "45": 2159231488.0, + "46": 2159231488.0, + "47": 2159231488.0, + "48": 2159231488.0, + "49": 2159231488.0, + "50": 2159231488.0, + "51": 2159231488.0, + "52": 2159231488.0, + "53": 2159231488.0, + "54": 2159231488.0, + "55": 2159231488.0, + "56": 2159231488.0, + "57": 2159231488.0, + "58": 2159231488.0, + "59": 2159231488.0, + "60": 2159231488.0, + "61": 2159231488.0, + "62": 2159231488.0, + "63": 2159231488.0, + "64": 2159231488.0, + "65": 2159231488.0, + "66": 2159231488.0, + "67": 2159231488.0, + "68": 2159231488.0, + "69": 2159231488.0, + "70": 2159231488.0, + "71": 2159231488.0, + "72": 2159231488.0, + "73": 2159231488.0, + "74": 2159231488.0, + "75": 2159231488.0, + "76": 2159231488.0, + "77": 2159231488.0, + "78": 2159231488.0, + "79": 2159231488.0, + "80": 2159231488.0, + "81": 2159231488.0, + "82": 2159231488.0, + "83": 2159231488.0, + "84": 2159231488.0, + "85": 2159231488.0, + "86": 2159231488.0, + "87": 2159231488.0, + "88": 2159231488.0, + "89": 2159231488.0, + "90": 2159231488.0, + "91": 2159231488.0, + "92": 2159231488.0, + "93": 2159231488.0, + "94": 2159231488.0, + "95": 2159231488.0, + "96": 2159231488.0, + "97": 2159231488.0, + "98": 2159231488.0, + "99": 2159231488.0, + "100": 2159231488.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.16725, - "3": 0.32945, - "4": 0.3326, - "5": 0.31532, - "6": 0.31313, - "7": 0.317, - "8": 0.30642, - "9": 0.30646, - "10": 0.3022, - "11": 0.30703, - "12": 0.30536, - "13": 0.30606, - "14": 0.30467, - "15": 0.3118, - "16": 0.30695, - "17": 0.30446, - "18": 0.30389, - "19": 0.30558, - "20": 0.30186, - "21": 0.30526, - "22": 0.3017, - "23": 0.30526, - "24": 0.30353, - "25": 0.30816, - "26": 0.30525, - "27": 0.30507, - "28": 0.30162, - "29": 0.29981, - "30": 0.30343, - "31": 0.30716, - "32": 0.29956, - "33": 0.29728, - "34": 0.29985, - "35": 0.29717, - "36": 0.30103, - "37": 0.30189, - "38": 0.30242, - "39": 0.30086, - "40": 0.3021, - "41": 0.30098, - "42": 0.29986, - "43": 0.30059, - "44": 0.30081, - "45": 0.29867, - "46": 0.30811, - "47": 0.3014, - "48": 0.3011, - "49": 0.30087, - "50": 0.3016, - "51": 0.36223, - "52": 2.63153, - "53": 0.30957, - "54": 0.30865, - "55": 0.30761, - "56": 0.30931, - "57": 0.30991, - "58": 0.31171, - "59": 0.30828, - "60": 0.30579, - "61": 0.30747, - "62": 0.31163, - "63": 0.3087, - "64": 0.31316, - "65": 0.30963, - "66": 0.3059, - "67": 0.30317, - "68": 0.30933, - "69": 0.30347, - "70": 0.30489, - "71": 0.31189, - "72": 0.31243, - "73": 0.3091, - "74": 0.31819, - "75": 0.33007, - "76": 0.3104, - "77": 0.31692, - "78": 0.31918, - "79": 0.32085, - "80": 0.32266, - "81": 0.33123, - "82": 0.31385, - "83": 0.36905, - "84": 0.33251, - "85": 0.33023, - "86": 0.33809, - "87": 0.30414, - "88": 0.31558, - "89": 0.3397, - "90": 0.32143, - "91": 0.33501, - "92": 0.32793, - "93": 0.34383, - "94": 0.33469, - "95": 0.3472, - "96": 0.37909, - "97": 0.34492, - "98": 0.34483, - "99": 0.33437, - "100": 0.34951 + "2": 5.1821, + "3": 0.33515, + "4": 0.32955, + "5": 0.32867, + "6": 0.32167, + "7": 0.32093, + "8": 0.31484, + "9": 0.3199, + "10": 0.32149, + "11": 0.32467, + "12": 0.32374, + "13": 0.31628, + "14": 0.32341, + "15": 0.32428, + "16": 0.31363, + "17": 0.31728, + "18": 0.3174, + "19": 0.31628, + "20": 0.31252, + "21": 0.31647, + "22": 0.31466, + "23": 0.31837, + "24": 0.31182, + "25": 0.31491, + "26": 0.3171, + "27": 0.32106, + "28": 0.31498, + "29": 0.31798, + "30": 0.31807, + "31": 0.31847, + "32": 0.31811, + "33": 0.31666, + "34": 0.31454, + "35": 0.31275, + "36": 0.31004, + "37": 0.31285, + "38": 0.32686, + "39": 0.31234, + "40": 0.31112, + "41": 0.31263, + "42": 0.31075, + "43": 0.31435, + "44": 0.31343, + "45": 0.31062, + "46": 0.31151, + "47": 0.30991, + "48": 0.31098, + "49": 0.31077, + "50": 0.31028, + "51": 0.3572, + "52": 2.64208, + "53": 0.31143, + "54": 0.31141, + "55": 0.31764, + "56": 0.3155, + "57": 0.30925, + "58": 0.31552, + "59": 0.3136, + "60": 0.31059, + "61": 0.31228, + "62": 0.3183, + "63": 0.31012, + "64": 0.31732, + "65": 0.31446, + "66": 0.31394, + "67": 0.31217, + "68": 0.31298, + "69": 0.31326, + "70": 0.31308, + "71": 0.31848, + "72": 0.31259, + "73": 0.31175, + "74": 0.32681, + "75": 0.33359, + "76": 0.32564, + "77": 0.32289, + "78": 0.32226, + "79": 0.32726, + "80": 0.33367, + "81": 0.31868, + "82": 0.31672, + "83": 0.37516, + "84": 0.3529, + "85": 0.34574, + "86": 0.34477, + "87": 0.31662, + "88": 0.36455, + "89": 0.35213, + "90": 0.33768, + "91": 0.33524, + "92": 0.33706, + "93": 0.32957, + "94": 0.34755, + "95": 0.36449, + "96": 0.36302, + "97": 0.36542, + "98": 0.37677, + "99": 0.34187, + "100": 0.35924 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 6dbbfe4def7..0e61235c41c 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.91066, - "2": 10.89646, - "3": 10.90792, - "4": 10.90307, - "5": 10.91195, - "6": 10.90646, - "7": 10.89838, - "8": 10.90122, - "9": 10.90026, - "10": 10.90331, - "11": 10.89166, - "12": 10.89988, - "13": 10.8849, - "14": 10.87998, - "15": 10.8728, - "16": 10.86504, - "17": 10.86869, - "18": 10.8571, - "19": 10.86119, - "20": 10.8188, - "21": 10.80335, - "22": 10.78966, - "23": 10.77604, - "24": 10.74885, - "25": 10.75645, - "26": 10.74017, - "27": 10.72496, - "28": 10.67041, - "29": 10.63542, - "30": 10.61272, - "31": 10.60548, - "32": 10.58806, - "33": 10.56518, - "34": 10.53433, - "35": 10.53075, - "36": 10.51437, - "37": 10.47627, - "38": 10.48915, - "39": 10.44952, - "40": 10.43444, - "41": 10.41455, - "42": 10.40199, - "43": 10.38762, - "44": 10.34257, - "45": 10.36188, - "46": 10.32335, - "47": 10.3067, - "48": 10.27172, - "49": 10.26262, - "50": 10.2613, - "51": 10.25781, - "52": 10.21493, - "53": 10.22583, - "54": 10.17883, - "55": 10.16505, - "56": 10.17871, - "57": 10.1692, - "58": 10.17718, - "59": 10.1302, - "60": 10.14386, - "61": 10.09356, - "62": 10.07265, - "63": 10.12749, - "64": 10.09284, - "65": 10.07125, - "66": 10.08262, - "67": 10.06175, - "68": 10.02605, - "69": 10.04908, - "70": 10.02759, - "71": 10.04607, - "72": 10.03034, - "73": 10.02624, - "74": 10.00701, - "75": 9.98506, - "76": 10.00662, - "77": 10.00431, - "78": 9.95602, - "79": 9.96789, - "80": 9.98225, - "81": 10.01113, - "82": 9.94293, - "83": 9.92061, - "84": 9.84992, - "85": 9.84827, - "86": 9.9353, - "87": 9.96845, - "88": 9.93373, - "89": 9.87283, - "90": 9.87687, - "91": 9.89287, - "92": 9.875, - "93": 9.81107, - "94": 9.89072, - "95": 9.87368, - "96": 9.85278, - "97": 9.7965, - "98": 9.83453, - "99": 9.87169, - "100": 9.76389 + "1": 10.91078, + "2": 10.89671, + "3": 10.9068, + "4": 10.90309, + "5": 10.91231, + "6": 10.90619, + "7": 10.89729, + "8": 10.90225, + "9": 10.89974, + "10": 10.90271, + "11": 10.89174, + "12": 10.89972, + "13": 10.88445, + "14": 10.88017, + "15": 10.8721, + "16": 10.86401, + "17": 10.86921, + "18": 10.85741, + "19": 10.8598, + "20": 10.81848, + "21": 10.80338, + "22": 10.78816, + "23": 10.77549, + "24": 10.74977, + "25": 10.75623, + "26": 10.74088, + "27": 10.72474, + "28": 10.6708, + "29": 10.63634, + "30": 10.61286, + "31": 10.60565, + "32": 10.58807, + "33": 10.56535, + "34": 10.53443, + "35": 10.53086, + "36": 10.5136, + "37": 10.47664, + "38": 10.48988, + "39": 10.44993, + "40": 10.43435, + "41": 10.41501, + "42": 10.40213, + "43": 10.38757, + "44": 10.34276, + "45": 10.36178, + "46": 10.32389, + "47": 10.30661, + "48": 10.27197, + "49": 10.26302, + "50": 10.26171, + "51": 10.25787, + "52": 10.2153, + "53": 10.22533, + "54": 10.17887, + "55": 10.16493, + "56": 10.17821, + "57": 10.1693, + "58": 10.17732, + "59": 10.13049, + "60": 10.14409, + "61": 10.09377, + "62": 10.07236, + "63": 10.1274, + "64": 10.09298, + "65": 10.07117, + "66": 10.0824, + "67": 10.06131, + "68": 10.02633, + "69": 10.04916, + "70": 10.02744, + "71": 10.04579, + "72": 10.03071, + "73": 10.02627, + "74": 10.00693, + "75": 9.9848, + "76": 10.00692, + "77": 10.00432, + "78": 9.95648, + "79": 9.96782, + "80": 9.98203, + "81": 10.01091, + "82": 9.94328, + "83": 9.92101, + "84": 9.84991, + "85": 9.84859, + "86": 9.93522, + "87": 9.96824, + "88": 9.93385, + "89": 9.87306, + "90": 9.87695, + "91": 9.89293, + "92": 9.87527, + "93": 9.81105, + "94": 9.8906, + "95": 9.87358, + "96": 9.85291, + "97": 9.79664, + "98": 9.83443, + "99": 9.87124, + "100": 9.76382 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 37001.0, - "2": 36732.0, - "3": 36500.0, - "4": 36497.0, - "5": 36308.0, - "6": 35804.0, - "7": 37128.0, - "8": 35982.0, - "9": 36339.0, - "10": 37420.0, - "11": 36825.0, - "12": 35918.0, - "13": 36509.0, - "14": 36718.0, - "15": 36144.0, - "16": 35879.0, - "17": 36557.0, - "18": 36406.0, - "19": 36689.0, - "20": 36332.0, - "21": 36165.0, - "22": 36186.0, - "23": 36662.0, - "24": 36628.0, - "25": 35283.0, - "26": 36188.0, - "27": 36438.0, - "28": 35557.0, - "29": 35655.0, - "30": 35347.0, - "31": 36045.0, - "32": 36682.0, - "33": 36030.0, - "34": 35613.0, - "35": 36767.0, - "36": 35693.0, - "37": 36422.0, - "38": 35167.0, - "39": 36773.0, - "40": 37229.0, - "41": 36203.0, - "42": 35749.0, - "43": 37587.0, - "44": 36237.0, - "45": 39444.0, - "46": 37852.0, - "47": 38394.0, - "48": 38946.0, - "49": 42609.0, - "50": 39507.0, - "51": 39090.0, - "52": 41821.0, - "53": 40567.0, - "54": 41610.0, - "55": 39318.0, - "56": 41334.0, - "57": 36899.0, - "58": 46231.0, - "59": 42246.0, - "60": 42784.0, - "61": 42495.0, - "62": 45631.0, - "63": 44361.0, - "64": 49155.0, - "65": 42460.0, - "66": 45659.0, - "67": 50043.0, - "68": 46333.0, - "69": 45862.0, - "70": 46950.0, - "71": 46684.0, - "72": 45346.0, - "73": 47664.0, - "74": 45501.0, - "75": 48985.0, - "76": 47531.0, - "77": 48431.0, - "78": 48651.0, - "79": 48300.0, - "80": 45717.0, - "81": 54912.0, - "82": 42473.0, - "83": 46906.0, - "84": 45830.0, - "85": 46189.0, - "86": 46183.0, - "87": 36768.0, - "88": 45219.0, - "89": 48522.0, - "90": 58149.0, - "91": 39290.0, - "92": 49092.0, - "93": 45067.0, - "94": 42178.0, - "95": 45548.0, - "96": 50098.0, - "97": 46773.0, - "98": 49620.0, - "99": 46056.0, - "100": 46551.0 + "1": 36671.0, + "2": 37197.0, + "3": 36559.0, + "4": 36852.0, + "5": 36703.0, + "6": 35840.0, + "7": 36983.0, + "8": 36383.0, + "9": 36544.0, + "10": 37416.0, + "11": 36558.0, + "12": 35768.0, + "13": 36071.0, + "14": 36725.0, + "15": 36398.0, + "16": 35673.0, + "17": 36265.0, + "18": 36819.0, + "19": 36835.0, + "20": 36479.0, + "21": 36112.0, + "22": 36404.0, + "23": 36252.0, + "24": 36731.0, + "25": 35963.0, + "26": 36050.0, + "27": 36455.0, + "28": 35396.0, + "29": 35673.0, + "30": 35389.0, + "31": 36231.0, + "32": 36304.0, + "33": 35334.0, + "34": 35822.0, + "35": 36789.0, + "36": 35934.0, + "37": 35999.0, + "38": 36004.0, + "39": 36913.0, + "40": 37742.0, + "41": 35768.0, + "42": 35680.0, + "43": 37957.0, + "44": 36315.0, + "45": 39567.0, + "46": 38316.0, + "47": 38689.0, + "48": 39256.0, + "49": 42420.0, + "50": 39683.0, + "51": 39232.0, + "52": 41791.0, + "53": 40285.0, + "54": 41889.0, + "55": 39269.0, + "56": 41161.0, + "57": 36797.0, + "58": 46655.0, + "59": 42337.0, + "60": 42800.0, + "61": 42084.0, + "62": 45422.0, + "63": 44419.0, + "64": 49741.0, + "65": 42303.0, + "66": 45995.0, + "67": 49650.0, + "68": 46704.0, + "69": 45620.0, + "70": 46489.0, + "71": 46470.0, + "72": 45280.0, + "73": 47307.0, + "74": 45593.0, + "75": 48865.0, + "76": 47482.0, + "77": 48068.0, + "78": 48859.0, + "79": 47823.0, + "80": 45845.0, + "81": 55013.0, + "82": 41851.0, + "83": 46834.0, + "84": 45934.0, + "85": 46257.0, + "86": 45740.0, + "87": 36471.0, + "88": 45175.0, + "89": 48591.0, + "90": 57540.0, + "91": 38958.0, + "92": 49460.0, + "93": 44750.0, + "94": 41945.0, + "95": 45537.0, + "96": 49807.0, + "97": 46836.0, + "98": 49860.0, + "99": 45566.0, + "100": 46454.0 } }, "mem-allocated-bytes": { @@ -218,104 +218,104 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254509056.0, - "2": 1254505984.0, - "3": 1254509056.0, + "1": 1254510592.0, + "2": 1254503936.0, + "3": 1254510080.0, "4": 1254507520.0, "5": 1254503936.0, - "6": 1254505472.0, - "7": 1254505472.0, + "6": 1254504960.0, + "7": 1254507008.0, "8": 1254505984.0, - "9": 1254509056.0, - "10": 1254509056.0, + "9": 1254510592.0, + "10": 1254508032.0, "11": 1254504960.0, "12": 1254507520.0, - "13": 1254508544.0, - "14": 1254507008.0, - "15": 1254505472.0, + "13": 1254508032.0, + "14": 1254506496.0, + "15": 1254507008.0, "16": 1254508032.0, "17": 1254507008.0, - "18": 1254507520.0, - "19": 1254504960.0, - "20": 1254509056.0, - "21": 1254507008.0, - "22": 1254505984.0, + "18": 1254506496.0, + "19": 1254505472.0, + "20": 1254509568.0, + "21": 1254505984.0, + "22": 1254506496.0, "23": 1254505472.0, - "24": 1254503936.0, - "25": 1254507520.0, - "26": 1254508032.0, - "27": 1254508544.0, - "28": 1254507008.0, - "29": 1254508544.0, - "30": 1254508032.0, - "31": 1254508544.0, - "32": 1254504960.0, + "24": 1254503424.0, + "25": 1254508032.0, + "26": 1254506496.0, + "27": 1254508032.0, + "28": 1254506496.0, + "29": 1254508032.0, + "30": 1254508544.0, + "31": 1254506496.0, + "32": 1254504448.0, "33": 1254506496.0, - "34": 1254504960.0, - "35": 1254504960.0, + "34": 1254506496.0, + "35": 1254504448.0, "36": 1254504960.0, - "37": 1254508032.0, - "38": 1254509056.0, - "39": 1254505984.0, - "40": 1254506496.0, - "41": 1254505472.0, + "37": 1254508544.0, + "38": 1254507520.0, + "39": 1254505472.0, + "40": 1254507008.0, + "41": 1254506496.0, "42": 1254504960.0, - "43": 1254507008.0, - "44": 1254503936.0, - "45": 1254504960.0, + "43": 1254506496.0, + "44": 1254504448.0, + "45": 1254504448.0, "46": 1254505984.0, "47": 1254506496.0, - "48": 1254506496.0, - "49": 1254508032.0, + "48": 1254507008.0, + "49": 1254507520.0, "50": 1254504448.0, - "51": 1254501376.0, + "51": 1254503424.0, "52": 1254506496.0, - "53": 1254503424.0, - "54": 1254503424.0, - "55": 1254505472.0, + "53": 1254504448.0, + "54": 1254504448.0, + "55": 1254506496.0, "56": 1254504448.0, "57": 1254507008.0, - "58": 1254508544.0, - "59": 1254505472.0, + "58": 1254507008.0, + "59": 1254504960.0, "60": 1254505984.0, - "61": 1254503424.0, + "61": 1254502912.0, "62": 1254505984.0, - "63": 1254506496.0, - "64": 1254505472.0, - "65": 1254507008.0, + "63": 1254505472.0, + "64": 1254505984.0, + "65": 1254505984.0, "66": 1254507008.0, - "67": 1254504448.0, - "68": 1254505984.0, - "69": 1254504960.0, - "70": 1254505472.0, + "67": 1254503936.0, + "68": 1254504448.0, + "69": 1254505472.0, + "70": 1254506496.0, "71": 1254502400.0, - "72": 1254504448.0, - "73": 1254502912.0, + "72": 1254503936.0, + "73": 1254502400.0, "74": 1254505984.0, "75": 1254505984.0, "76": 1254505472.0, - "77": 1254505984.0, - "78": 1254502912.0, - "79": 1254506496.0, + "77": 1254505472.0, + "78": 1254501376.0, + "79": 1254504960.0, "80": 1254505984.0, - "81": 1254505472.0, + "81": 1254504960.0, "82": 1254505472.0, - "83": 1254503424.0, + "83": 1254504448.0, "84": 1254505984.0, - "85": 1254504960.0, - "86": 1254506496.0, - "87": 1254504448.0, - "88": 1254502400.0, - "89": 1254509568.0, + "85": 1254503424.0, + "86": 1254505984.0, + "87": 1254502912.0, + "88": 1254501376.0, + "89": 1254508032.0, "90": 1254507008.0, - "91": 1254503936.0, + "91": 1254504448.0, "92": 1254505984.0, "93": 1254507008.0, - "94": 1254507520.0, - "95": 1254511104.0, - "96": 1254504448.0, - "97": 1254509568.0, - "98": 1254507520.0, + "94": 1254506496.0, + "95": 1254513664.0, + "96": 1254503936.0, + "97": 1254509056.0, + "98": 1254508032.0, "99": 1254508032.0, "100": 1254509056.0 } @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2065569280.0, - "2": 2546124800.0, - "3": 2546124800.0, - "4": 2546124800.0, - "5": 2546257920.0, - "6": 2546257920.0, - "7": 2546452480.0, - "8": 2546452480.0, - "9": 2546452480.0, - "10": 2546452480.0, - "11": 2546452480.0, - "12": 2546452480.0, - "13": 2546452480.0, - "14": 2546452480.0, - "15": 2546452480.0, - "16": 2546452480.0, - "17": 2546452480.0, - "18": 2546452480.0, - "19": 2546452480.0, - "20": 2546452480.0, - "21": 2546452480.0, - "22": 2546452480.0, - "23": 2546452480.0, - "24": 2546452480.0, - "25": 2546452480.0, - "26": 2546452480.0, - "27": 2546452480.0, - "28": 2546452480.0, - "29": 2546452480.0, - "30": 2546452480.0, - "31": 2546757632.0, - "32": 2546757632.0, - "33": 2546757632.0, - "34": 2546757632.0, - "35": 2546757632.0, - "36": 2546757632.0, - "37": 2546757632.0, - "38": 2546757632.0, - "39": 2546757632.0, - "40": 2546757632.0, - "41": 2546757632.0, - "42": 2546757632.0, - "43": 2546757632.0, - "44": 2546757632.0, - "45": 2546757632.0, - "46": 2546757632.0, - "47": 2546757632.0, - "48": 2546757632.0, - "49": 2546757632.0, - "50": 2546757632.0, - "51": 2546757632.0, - "52": 2546757632.0, - "53": 2546757632.0, - "54": 2546757632.0, - "55": 2546757632.0, - "56": 2546757632.0, - "57": 2546757632.0, - "58": 2546757632.0, - "59": 2546757632.0, - "60": 2546757632.0, - "61": 2546757632.0, - "62": 2546757632.0, - "63": 2546757632.0, - "64": 2546757632.0, - "65": 2546757632.0, - "66": 2546757632.0, - "67": 2546757632.0, - "68": 2546757632.0, - "69": 2546757632.0, - "70": 2546757632.0, - "71": 2546757632.0, - "72": 2546757632.0, - "73": 2546757632.0, - "74": 2546757632.0, - "75": 2546757632.0, - "76": 2546757632.0, - "77": 2546757632.0, - "78": 2546757632.0, - "79": 2546757632.0, - "80": 2546757632.0, - "81": 2546757632.0, - "82": 2546757632.0, - "83": 2546757632.0, - "84": 2546757632.0, - "85": 2546757632.0, - "86": 2546757632.0, - "87": 2546757632.0, - "88": 2546757632.0, - "89": 2546757632.0, - "90": 2546757632.0, - "91": 2546757632.0, - "92": 2546757632.0, - "93": 2548838912.0, - "94": 2548838912.0, - "95": 2549586432.0, - "96": 2549586432.0, - "97": 2549698048.0, - "98": 2549698048.0, - "99": 2549698048.0, - "100": 2553959936.0 + "1": 2065977344.0, + "2": 2545691648.0, + "3": 2546333696.0, + "4": 2546333696.0, + "5": 2546333696.0, + "6": 2546333696.0, + "7": 2546546688.0, + "8": 2546546688.0, + "9": 2546546688.0, + "10": 2546546688.0, + "11": 2546546688.0, + "12": 2546546688.0, + "13": 2546546688.0, + "14": 2546546688.0, + "15": 2546546688.0, + "16": 2546546688.0, + "17": 2546546688.0, + "18": 2546546688.0, + "19": 2546546688.0, + "20": 2546546688.0, + "21": 2546546688.0, + "22": 2546546688.0, + "23": 2546546688.0, + "24": 2546625024.0, + "25": 2546625024.0, + "26": 2546625024.0, + "27": 2546625024.0, + "28": 2546625024.0, + "29": 2546625024.0, + "30": 2546625024.0, + "31": 2546924544.0, + "32": 2546924544.0, + "33": 2546924544.0, + "34": 2546924544.0, + "35": 2546924544.0, + "36": 2546924544.0, + "37": 2546924544.0, + "38": 2546924544.0, + "39": 2546924544.0, + "40": 2546924544.0, + "41": 2546924544.0, + "42": 2546924544.0, + "43": 2546924544.0, + "44": 2546924544.0, + "45": 2546924544.0, + "46": 2546924544.0, + "47": 2546924544.0, + "48": 2546924544.0, + "49": 2546924544.0, + "50": 2546924544.0, + "51": 2546924544.0, + "52": 2546924544.0, + "53": 2546924544.0, + "54": 2546924544.0, + "55": 2546924544.0, + "56": 2546924544.0, + "57": 2546924544.0, + "58": 2546924544.0, + "59": 2546924544.0, + "60": 2546924544.0, + "61": 2546924544.0, + "62": 2546924544.0, + "63": 2546924544.0, + "64": 2546924544.0, + "65": 2546924544.0, + "66": 2546924544.0, + "67": 2546924544.0, + "68": 2546924544.0, + "69": 2546924544.0, + "70": 2546924544.0, + "71": 2546924544.0, + "72": 2546924544.0, + "73": 2546924544.0, + "74": 2546924544.0, + "75": 2546924544.0, + "76": 2546924544.0, + "77": 2546924544.0, + "78": 2546924544.0, + "79": 2546924544.0, + "80": 2546924544.0, + "81": 2546924544.0, + "82": 2546924544.0, + "83": 2546924544.0, + "84": 2546924544.0, + "85": 2548597248.0, + "86": 2548597248.0, + "87": 2548597248.0, + "88": 2548597248.0, + "89": 2548597248.0, + "90": 2548597248.0, + "91": 2548597248.0, + "92": 2548597248.0, + "93": 2550968832.0, + "94": 2550968832.0, + "95": 2550968832.0, + "96": 2550968832.0, + "97": 2551127552.0, + "98": 2551127552.0, + "99": 2551127552.0, + "100": 2552623616.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.35065, - "3": 1.49589, - "4": 1.34581, - "5": 0.80643, - "6": 1.43894, - "7": 1.37891, - "8": 1.49563, - "9": 1.95785, - "10": 0.85481, - "11": 1.33899, - "12": 1.43755, - "13": 1.19509, - "14": 1.83328, - "15": 0.98032, - "16": 1.40874, - "17": 1.66864, - "18": 1.5365, - "19": 1.30779, - "20": 1.79944, - "21": 0.71488, - "22": 1.32647, - "23": 1.24779, - "24": 2.20621, - "25": 1.10326, - "26": 0.96542, - "27": 1.29006, - "28": 1.41795, - "29": 1.89483, - "30": 0.79813, - "31": 0.97715, - "32": 2.08206, - "33": 1.21236, - "34": 0.71896, - "35": 1.46986, - "36": 1.39013, - "37": 1.24449, - "38": 0.73381, - "39": 1.35307, - "40": 1.7201, - "41": 1.25122, - "42": 1.33799, - "43": 1.08317, - "44": 1.41303, - "45": 1.08191, - "46": 0.79454, - "47": 1.23184, - "48": 1.40187, - "49": 1.10435, - "50": 1.36801, - "51": 0.84116, - "52": 1.02029, - "53": 1.39142, - "54": 2.03947, - "55": 1.50361, - "56": 1.31223, - "57": 0.93054, - "58": 1.61309, - "59": 1.19898, - "60": 1.08312, - "61": 1.63845, - "62": 1.43415, - "63": 1.40495, - "64": 1.31806, - "65": 1.06803, - "66": 1.3885, - "67": 1.59941, - "68": 1.09038, - "69": 1.70706, - "70": 1.28046, - "71": 1.41662, - "72": 1.50138, - "73": 1.2025, - "74": 0.91636, - "75": 1.18148, - "76": 1.36065, - "77": 1.41942, - "78": 1.33114, - "79": 1.88785, - "80": 1.08232, - "81": 1.31472, - "82": 1.68157, - "83": 1.15155, - "84": 1.18209, - "85": 0.9033, - "86": 1.09546, - "87": 1.81244, - "88": 1.1479, - "89": 1.00561, - "90": 1.55058, - "91": 0.97663, - "92": 1.58765, - "93": 1.01357, - "94": 1.08504, - "95": 1.09183, - "96": 1.88433, - "97": 1.03535, - "98": 0.77708, - "99": 1.69156, - "100": 1.06124 + "2": 5.08084, + "3": 1.2241, + "4": 1.10762, + "5": 0.70762, + "6": 1.26303, + "7": 1.08459, + "8": 1.32569, + "9": 1.40813, + "10": 0.91633, + "11": 1.10189, + "12": 1.15335, + "13": 0.70869, + "14": 1.56963, + "15": 0.76781, + "16": 1.09239, + "17": 1.02387, + "18": 1.01316, + "19": 1.35792, + "20": 0.93147, + "21": 0.97619, + "22": 1.04357, + "23": 1.06241, + "24": 1.57092, + "25": 0.64516, + "26": 0.97019, + "27": 0.98101, + "28": 1.0787, + "29": 0.95435, + "30": 0.8968, + "31": 0.79525, + "32": 1.72006, + "33": 0.98605, + "34": 0.61553, + "35": 1.21, + "36": 1.22026, + "37": 1.07002, + "38": 0.61571, + "39": 1.13348, + "40": 1.42486, + "41": 1.02872, + "42": 1.19202, + "43": 0.98969, + "44": 1.29601, + "45": 0.68946, + "46": 1.07996, + "47": 1.10518, + "48": 1.02709, + "49": 1.06183, + "50": 1.10737, + "51": 0.68569, + "52": 0.88824, + "53": 1.25525, + "54": 1.30481, + "55": 1.17559, + "56": 1.00159, + "57": 0.88713, + "58": 1.32751, + "59": 0.9028, + "60": 0.86119, + "61": 1.21504, + "62": 1.23664, + "63": 0.99694, + "64": 1.22317, + "65": 0.82966, + "66": 1.20659, + "67": 1.36925, + "68": 0.97466, + "69": 1.43404, + "70": 1.12194, + "71": 1.10993, + "72": 1.27258, + "73": 1.09257, + "74": 0.69788, + "75": 1.15638, + "76": 1.1445, + "77": 1.39873, + "78": 1.32277, + "79": 1.74507, + "80": 1.03107, + "81": 1.03289, + "82": 1.42932, + "83": 1.10664, + "84": 1.20253, + "85": 0.75253, + "86": 1.00083, + "87": 1.8368, + "88": 1.03179, + "89": 0.88933, + "90": 1.16981, + "91": 1.01329, + "92": 1.49931, + "93": 0.81274, + "94": 1.13907, + "95": 1.0751, + "96": 1.1469, + "97": 0.77969, + "98": 1.3287, + "99": 1.6143, + "100": 1.154 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json index 1aa04fb6e79..416c7c0b51a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.91731, - "2": 10.90352, - "3": 10.9039, - "4": 10.89709, - "5": 10.91817, - "6": 10.90942, - "7": 10.8988, - "8": 10.90009, - "9": 10.90396, - "10": 10.90833, - "11": 10.89594, - "12": 10.90288, - "13": 10.88181, - "14": 10.88133, - "15": 10.87556, - "16": 10.86702, - "17": 10.86507, - "18": 10.85912, - "19": 10.86247, - "20": 10.8161, - "21": 10.80044, - "22": 10.7913, - "23": 10.78124, - "24": 10.74776, - "25": 10.75813, - "26": 10.73675, - "27": 10.72212, - "28": 10.67475, - "29": 10.63486, - "30": 10.61818, - "31": 10.60416, - "32": 10.58976, - "33": 10.5619, - "34": 10.53591, - "35": 10.52892, - "36": 10.51876, - "37": 10.48228, - "38": 10.48829, - "39": 10.45257, - "40": 10.43499, - "41": 10.41456, - "42": 10.40579, - "43": 10.38798, - "44": 10.34194, - "45": 10.36245, - "46": 10.32565, - "47": 10.30551, - "48": 10.27423, - "49": 10.26214, - "50": 10.26183, - "51": 10.25946, - "52": 10.2173, - "53": 10.22317, - "54": 10.17924, - "55": 10.16572, - "56": 10.17818, - "57": 10.16906, - "58": 10.17739, - "59": 10.13026, - "60": 10.14423, - "61": 10.09505, + "1": 10.91589, + "2": 10.90332, + "3": 10.90347, + "4": 10.89774, + "5": 10.91869, + "6": 10.90876, + "7": 10.89808, + "8": 10.90041, + "9": 10.90403, + "10": 10.90843, + "11": 10.8958, + "12": 10.90348, + "13": 10.88305, + "14": 10.88331, + "15": 10.87671, + "16": 10.86763, + "17": 10.86407, + "18": 10.85889, + "19": 10.86376, + "20": 10.81635, + "21": 10.80059, + "22": 10.79096, + "23": 10.78086, + "24": 10.74863, + "25": 10.75894, + "26": 10.73695, + "27": 10.72356, + "28": 10.67453, + "29": 10.6352, + "30": 10.61814, + "31": 10.60455, + "32": 10.58898, + "33": 10.56262, + "34": 10.53561, + "35": 10.52922, + "36": 10.51896, + "37": 10.48236, + "38": 10.48771, + "39": 10.45244, + "40": 10.43529, + "41": 10.41515, + "42": 10.40591, + "43": 10.388, + "44": 10.34146, + "45": 10.36294, + "46": 10.32607, + "47": 10.30568, + "48": 10.27378, + "49": 10.26213, + "50": 10.26235, + "51": 10.25989, + "52": 10.2171, + "53": 10.22323, + "54": 10.17928, + "55": 10.16585, + "56": 10.17764, + "57": 10.16944, + "58": 10.17725, + "59": 10.13037, + "60": 10.14429, + "61": 10.09562, "62": 10.0697, - "63": 10.12716, - "64": 10.09493, - "65": 10.07355, - "66": 10.08342, - "67": 10.06417, - "68": 10.02678, - "69": 10.05099, - "70": 10.02711, - "71": 10.04652, - "72": 10.03304, - "73": 10.02478, - "74": 10.00613, - "75": 9.98662, - "76": 10.00609, - "77": 10.00697, - "78": 9.95712, - "79": 9.96952, - "80": 9.98346, - "81": 10.01077, - "82": 9.94178, - "83": 9.92025, - "84": 9.85032, - "85": 9.84755, - "86": 9.93296, - "87": 9.96442, - "88": 9.93276, - "89": 9.87215, + "63": 10.12696, + "64": 10.09479, + "65": 10.07397, + "66": 10.08364, + "67": 10.06375, + "68": 10.02703, + "69": 10.05115, + "70": 10.02703, + "71": 10.04664, + "72": 10.0328, + "73": 10.02496, + "74": 10.00629, + "75": 9.98643, + "76": 10.00588, + "77": 10.00712, + "78": 9.95731, + "79": 9.96935, + "80": 9.98348, + "81": 10.0112, + "82": 9.94191, + "83": 9.92046, + "84": 9.8503, + "85": 9.84797, + "86": 9.93316, + "87": 9.96433, + "88": 9.93294, + "89": 9.87213, "90": 9.87714, - "91": 9.89315, - "92": 9.87583, - "93": 9.81093, - "94": 9.89075, - "95": 9.87251, - "96": 9.85407, - "97": 9.79978, - "98": 9.83405, - "99": 9.8714, - "100": 9.76512 + "91": 9.89324, + "92": 9.87604, + "93": 9.81103, + "94": 9.89137, + "95": 9.87307, + "96": 9.85376, + "97": 9.79965, + "98": 9.83401, + "99": 9.87123, + "100": 9.7653 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36422.0, - "2": 36679.0, - "3": 36905.0, - "4": 36533.0, - "5": 36340.0, - "6": 36090.0, - "7": 36920.0, - "8": 36338.0, - "9": 36798.0, - "10": 37298.0, - "11": 36782.0, - "12": 36234.0, - "13": 36775.0, - "14": 36638.0, - "15": 36024.0, - "16": 35653.0, - "17": 36755.0, - "18": 36446.0, - "19": 36807.0, - "20": 36388.0, - "21": 36292.0, - "22": 36336.0, - "23": 36729.0, - "24": 36627.0, - "25": 35025.0, - "26": 36264.0, - "27": 36341.0, - "28": 35309.0, - "29": 35514.0, - "30": 35251.0, - "31": 36280.0, - "32": 36533.0, - "33": 35741.0, - "34": 36250.0, - "35": 36049.0, - "36": 35497.0, - "37": 36212.0, - "38": 35415.0, - "39": 36529.0, - "40": 37543.0, - "41": 36221.0, - "42": 35514.0, - "43": 37348.0, - "44": 36680.0, - "45": 39589.0, - "46": 38348.0, - "47": 38309.0, - "48": 39160.0, - "49": 42077.0, - "50": 39572.0, - "51": 38572.0, - "52": 41639.0, - "53": 40433.0, - "54": 42084.0, - "55": 39430.0, - "56": 41336.0, - "57": 36885.0, - "58": 46237.0, - "59": 42482.0, - "60": 42940.0, - "61": 42238.0, - "62": 45567.0, - "63": 44114.0, - "64": 49016.0, - "65": 42027.0, - "66": 46057.0, - "67": 49536.0, - "68": 46490.0, - "69": 46074.0, - "70": 47203.0, - "71": 46761.0, - "72": 45627.0, - "73": 47213.0, - "74": 45958.0, - "75": 48735.0, - "76": 47980.0, - "77": 48043.0, - "78": 49253.0, - "79": 47579.0, - "80": 45878.0, - "81": 54786.0, - "82": 42125.0, - "83": 46716.0, - "84": 45239.0, - "85": 45470.0, - "86": 45817.0, - "87": 36232.0, - "88": 45348.0, - "89": 48564.0, - "90": 56854.0, - "91": 39336.0, - "92": 49663.0, - "93": 45007.0, - "94": 42457.0, - "95": 44804.0, - "96": 49933.0, - "97": 46138.0, - "98": 50480.0, - "99": 45746.0, - "100": 47024.0 + "1": 36227.0, + "2": 37201.0, + "3": 36774.0, + "4": 36570.0, + "5": 36439.0, + "6": 35792.0, + "7": 37127.0, + "8": 36144.0, + "9": 36928.0, + "10": 37176.0, + "11": 37010.0, + "12": 35585.0, + "13": 37190.0, + "14": 36821.0, + "15": 35771.0, + "16": 36427.0, + "17": 36272.0, + "18": 36691.0, + "19": 36485.0, + "20": 36518.0, + "21": 36020.0, + "22": 36224.0, + "23": 36375.0, + "24": 36945.0, + "25": 35124.0, + "26": 36177.0, + "27": 36186.0, + "28": 35639.0, + "29": 35438.0, + "30": 35209.0, + "31": 36595.0, + "32": 36317.0, + "33": 35600.0, + "34": 35742.0, + "35": 36404.0, + "36": 35231.0, + "37": 36629.0, + "38": 35718.0, + "39": 36298.0, + "40": 37761.0, + "41": 36285.0, + "42": 35162.0, + "43": 37644.0, + "44": 36847.0, + "45": 39443.0, + "46": 38325.0, + "47": 38405.0, + "48": 39108.0, + "49": 42153.0, + "50": 39482.0, + "51": 38581.0, + "52": 41497.0, + "53": 40023.0, + "54": 41751.0, + "55": 39497.0, + "56": 41852.0, + "57": 37239.0, + "58": 46019.0, + "59": 42212.0, + "60": 42885.0, + "61": 42226.0, + "62": 45308.0, + "63": 44346.0, + "64": 49291.0, + "65": 41989.0, + "66": 46316.0, + "67": 50078.0, + "68": 46321.0, + "69": 46309.0, + "70": 46910.0, + "71": 46636.0, + "72": 45539.0, + "73": 47682.0, + "74": 45594.0, + "75": 49085.0, + "76": 48381.0, + "77": 48005.0, + "78": 48958.0, + "79": 48068.0, + "80": 45362.0, + "81": 54972.0, + "82": 41961.0, + "83": 46646.0, + "84": 45351.0, + "85": 45681.0, + "86": 45789.0, + "87": 36100.0, + "88": 45173.0, + "89": 48281.0, + "90": 56672.0, + "91": 38947.0, + "92": 48946.0, + "93": 45289.0, + "94": 42563.0, + "95": 45023.0, + "96": 49854.0, + "97": 46352.0, + "98": 50691.0, + "99": 45749.0, + "100": 46988.0 } }, "mem-allocated-bytes": { @@ -219,105 +219,105 @@ "step_interval": 1, "values": { "1": 1254505472.0, - "2": 1254504960.0, - "3": 1254507008.0, + "2": 1254504448.0, + "3": 1254505984.0, "4": 1254506496.0, - "5": 1254504448.0, - "6": 1254502400.0, - "7": 1254505984.0, - "8": 1254508032.0, - "9": 1254508544.0, - "10": 1254508032.0, - "11": 1254505984.0, - "12": 1254505984.0, + "5": 1254504960.0, + "6": 1254502912.0, + "7": 1254505472.0, + "8": 1254507008.0, + "9": 1254508032.0, + "10": 1254507520.0, + "11": 1254506496.0, + "12": 1254507008.0, "13": 1254506496.0, - "14": 1254507008.0, + "14": 1254506496.0, "15": 1254507008.0, - "16": 1254509056.0, + "16": 1254507008.0, "17": 1254505984.0, - "18": 1254507520.0, - "19": 1254508032.0, - "20": 1254507008.0, + "18": 1254508032.0, + "19": 1254508544.0, + "20": 1254508544.0, "21": 1254505472.0, - "22": 1254502400.0, - "23": 1254506496.0, - "24": 1254505984.0, + "22": 1254501376.0, + "23": 1254505472.0, + "24": 1254507008.0, "25": 1254501376.0, - "26": 1254507008.0, - "27": 1254507008.0, + "26": 1254508544.0, + "27": 1254505984.0, "28": 1254503424.0, "29": 1254505984.0, - "30": 1254506496.0, - "31": 1254510080.0, - "32": 1254503424.0, + "30": 1254507008.0, + "31": 1254508544.0, + "32": 1254502912.0, "33": 1254508544.0, - "34": 1254505984.0, - "35": 1254505472.0, - "36": 1254505472.0, + "34": 1254504448.0, + "35": 1254504960.0, + "36": 1254504448.0, "37": 1254503936.0, "38": 1254505472.0, - "39": 1254504448.0, - "40": 1254504960.0, - "41": 1254507520.0, + "39": 1254503936.0, + "40": 1254505472.0, + "41": 1254506496.0, "42": 1254504960.0, - "43": 1254505984.0, - "44": 1254505472.0, - "45": 1254505984.0, - "46": 1254505984.0, + "43": 1254505472.0, + "44": 1254504960.0, + "45": 1254506496.0, + "46": 1254505472.0, "47": 1254505984.0, - "48": 1254505472.0, - "49": 1254506496.0, - "50": 1254504960.0, - "51": 1254506496.0, - "52": 1254507008.0, - "53": 1254504960.0, - "54": 1254503936.0, - "55": 1254504960.0, - "56": 1254503424.0, + "48": 1254505984.0, + "49": 1254507520.0, + "50": 1254503936.0, + "51": 1254508032.0, + "52": 1254506496.0, + "53": 1254504448.0, + "54": 1254503424.0, + "55": 1254505984.0, + "56": 1254502912.0, "57": 1254506496.0, - "58": 1254508032.0, - "59": 1254503936.0, - "60": 1254503936.0, - "61": 1254501376.0, - "62": 1254505472.0, + "58": 1254507520.0, + "59": 1254504448.0, + "60": 1254503424.0, + "61": 1254501888.0, + "62": 1254503936.0, "63": 1254504448.0, "64": 1254509568.0, - "65": 1254503424.0, - "66": 1254503424.0, + "65": 1254503936.0, + "66": 1254504960.0, "67": 1254504448.0, "68": 1254504448.0, "69": 1254504960.0, - "70": 1254502400.0, - "71": 1254506496.0, + "70": 1254503424.0, + "71": 1254505984.0, "72": 1254504448.0, "73": 1254504960.0, - "74": 1254502912.0, - "75": 1254502912.0, - "76": 1254506496.0, - "77": 1254503936.0, - "78": 1254503424.0, - "79": 1254506496.0, - "80": 1254505472.0, - "81": 1254502400.0, + "74": 1254502400.0, + "75": 1254502400.0, + "76": 1254508032.0, + "77": 1254504960.0, + "78": 1254502912.0, + "79": 1254504448.0, + "80": 1254505984.0, + "81": 1254502912.0, "82": 1254505984.0, - "83": 1254502400.0, - "84": 1254507008.0, - "85": 1254501888.0, + "83": 1254503424.0, + "84": 1254508032.0, + "85": 1254502912.0, "86": 1254504960.0, - "87": 1254503424.0, - "88": 1254503936.0, - "89": 1254506496.0, - "90": 1254507520.0, - "91": 1254503936.0, - "92": 1254505984.0, - "93": 1254502912.0, - "94": 1254503936.0, + "87": 1254502912.0, + "88": 1254502912.0, + "89": 1254507520.0, + "90": 1254508032.0, + "91": 1254504448.0, + "92": 1254507008.0, + "93": 1254503424.0, + "94": 1254503424.0, "95": 1254512640.0, "96": 1254504960.0, - "97": 1254507008.0, - "98": 1254509568.0, - "99": 1254505984.0, - "100": 1254513152.0 + "97": 1254507520.0, + "98": 1254509056.0, + "99": 1254507008.0, + "100": 1254513664.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2064222208.0, - "2": 2547451904.0, - "3": 2547451904.0, - "4": 2547745280.0, - "5": 2547745280.0, - "6": 2547745280.0, - "7": 2547745280.0, - "8": 2547745280.0, - "9": 2547745280.0, - "10": 2547745280.0, - "11": 2547819520.0, - "12": 2547819520.0, - "13": 2547819520.0, - "14": 2547819520.0, - "15": 2547819520.0, - "16": 2547819520.0, - "17": 2547819520.0, - "18": 2547819520.0, - "19": 2547819520.0, - "20": 2547819520.0, - "21": 2547819520.0, - "22": 2547819520.0, - "23": 2547819520.0, - "24": 2547819520.0, - "25": 2547819520.0, - "26": 2547819520.0, - "27": 2547819520.0, - "28": 2547819520.0, - "29": 2547819520.0, - "30": 2547819520.0, - "31": 2547819520.0, - "32": 2547819520.0, - "33": 2547819520.0, - "34": 2547819520.0, - "35": 2547819520.0, - "36": 2547819520.0, - "37": 2547819520.0, - "38": 2547819520.0, - "39": 2547819520.0, - "40": 2547819520.0, - "41": 2547819520.0, - "42": 2547819520.0, - "43": 2547819520.0, - "44": 2547819520.0, - "45": 2547819520.0, - "46": 2547819520.0, - "47": 2548183552.0, - "48": 2548183552.0, - "49": 2548183552.0, - "50": 2548183552.0, - "51": 2548183552.0, - "52": 2548183552.0, - "53": 2548183552.0, - "54": 2548183552.0, - "55": 2548183552.0, - "56": 2548183552.0, - "57": 2548183552.0, - "58": 2548183552.0, - "59": 2548183552.0, - "60": 2548183552.0, - "61": 2548183552.0, - "62": 2548183552.0, - "63": 2548183552.0, - "64": 2548183552.0, - "65": 2548183552.0, - "66": 2548183552.0, - "67": 2548183552.0, - "68": 2548183552.0, - "69": 2548183552.0, - "70": 2548183552.0, - "71": 2548183552.0, - "72": 2548183552.0, - "73": 2548183552.0, - "74": 2548183552.0, - "75": 2548183552.0, - "76": 2548183552.0, - "77": 2548183552.0, - "78": 2548183552.0, - "79": 2548183552.0, - "80": 2548183552.0, - "81": 2548183552.0, - "82": 2548183552.0, - "83": 2548183552.0, - "84": 2548183552.0, - "85": 2548183552.0, - "86": 2548183552.0, - "87": 2548183552.0, - "88": 2548183552.0, - "89": 2548183552.0, - "90": 2548183552.0, - "91": 2548183552.0, - "92": 2548183552.0, - "93": 2548183552.0, - "94": 2548183552.0, - "95": 2551813120.0, - "96": 2551813120.0, - "97": 2551813120.0, - "98": 2551813120.0, - "99": 2551813120.0, - "100": 2557905408.0 + "1": 2064315392.0, + "2": 2545652736.0, + "3": 2546057728.0, + "4": 2547006464.0, + "5": 2547006464.0, + "6": 2547006464.0, + "7": 2547006464.0, + "8": 2547006464.0, + "9": 2547006464.0, + "10": 2547006464.0, + "11": 2547006464.0, + "12": 2547006464.0, + "13": 2547006464.0, + "14": 2547006464.0, + "15": 2547006464.0, + "16": 2547006464.0, + "17": 2547006464.0, + "18": 2547006464.0, + "19": 2547006464.0, + "20": 2547006464.0, + "21": 2547006464.0, + "22": 2547006464.0, + "23": 2547006464.0, + "24": 2547006464.0, + "25": 2547006464.0, + "26": 2547006464.0, + "27": 2547006464.0, + "28": 2547006464.0, + "29": 2547006464.0, + "30": 2547006464.0, + "31": 2547006464.0, + "32": 2547006464.0, + "33": 2547006464.0, + "34": 2547006464.0, + "35": 2547006464.0, + "36": 2547006464.0, + "37": 2547006464.0, + "38": 2547006464.0, + "39": 2547006464.0, + "40": 2547006464.0, + "41": 2547006464.0, + "42": 2547006464.0, + "43": 2547006464.0, + "44": 2547006464.0, + "45": 2547006464.0, + "46": 2547006464.0, + "47": 2547006464.0, + "48": 2547006464.0, + "49": 2547006464.0, + "50": 2547006464.0, + "51": 2547006464.0, + "52": 2547006464.0, + "53": 2547006464.0, + "54": 2547006464.0, + "55": 2547006464.0, + "56": 2547006464.0, + "57": 2547006464.0, + "58": 2547006464.0, + "59": 2547006464.0, + "60": 2547006464.0, + "61": 2547006464.0, + "62": 2547006464.0, + "63": 2547006464.0, + "64": 2547006464.0, + "65": 2547006464.0, + "66": 2547006464.0, + "67": 2547006464.0, + "68": 2547006464.0, + "69": 2547006464.0, + "70": 2547006464.0, + "71": 2547006464.0, + "72": 2547006464.0, + "73": 2547006464.0, + "74": 2547006464.0, + "75": 2547006464.0, + "76": 2547006464.0, + "77": 2547006464.0, + "78": 2547006464.0, + "79": 2547006464.0, + "80": 2547006464.0, + "81": 2547006464.0, + "82": 2547006464.0, + "83": 2547006464.0, + "84": 2547006464.0, + "85": 2547006464.0, + "86": 2547006464.0, + "87": 2547006464.0, + "88": 2547006464.0, + "89": 2547006464.0, + "90": 2547006464.0, + "91": 2547474944.0, + "92": 2547474944.0, + "93": 2547474944.0, + "94": 2547474944.0, + "95": 2552599552.0, + "96": 2552599552.0, + "97": 2552599552.0, + "98": 2552599552.0, + "99": 2552599552.0, + "100": 2555065344.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.94692, - "3": 0.39374, - "4": 0.3842, - "5": 0.36988, - "6": 0.37797, - "7": 0.37172, - "8": 0.37093, - "9": 0.37604, - "10": 0.36954, - "11": 0.3688, - "12": 0.37204, - "13": 0.36306, - "14": 0.37283, - "15": 0.37318, - "16": 0.37125, - "17": 0.37306, - "18": 0.37008, - "19": 0.36747, - "20": 0.3699, - "21": 0.36721, - "22": 0.36941, - "23": 0.3703, - "24": 0.36971, - "25": 0.37269, - "26": 0.37442, - "27": 0.37359, - "28": 0.37397, - "29": 0.37473, - "30": 0.36489, - "31": 0.3678, - "32": 0.37099, - "33": 0.3675, - "34": 0.37317, - "35": 0.37318, - "36": 0.36474, - "37": 0.36455, - "38": 0.36854, - "39": 0.37178, - "40": 0.37108, - "41": 0.3773, - "42": 0.37836, - "43": 0.37171, - "44": 0.42276, - "45": 0.45488, - "46": 0.54196, - "47": 0.3736, - "48": 0.36864, - "49": 0.36923, - "50": 0.41777, - "51": 0.54436, - "52": 0.38619, - "53": 0.36947, - "54": 0.38345, - "55": 0.42913, - "56": 0.3728, - "57": 0.37395, - "58": 0.36863, - "59": 0.37623, - "60": 0.37899, - "61": 0.37369, - "62": 0.37197, - "63": 0.37268, - "64": 0.36933, - "65": 0.37068, - "66": 0.37343, - "67": 0.37253, - "68": 0.37331, - "69": 0.37376, - "70": 0.37165, - "71": 0.37998, - "72": 0.38313, - "73": 0.37481, - "74": 0.37724, - "75": 0.3745, - "76": 0.37916, - "77": 0.37341, - "78": 0.37715, - "79": 0.37486, - "80": 0.37551, - "81": 0.37368, - "82": 0.37218, - "83": 0.38333, - "84": 0.37439, - "85": 0.37872, - "86": 0.3731, - "87": 0.37113, - "88": 0.38104, - "89": 0.38597, - "90": 0.3846, - "91": 0.38888, - "92": 0.38405, - "93": 0.3875, - "94": 0.38885, - "95": 0.39876, - "96": 0.40047, - "97": 0.39001, - "98": 0.39211, - "99": 0.38241, - "100": 0.40784 + "2": 10.45476, + "3": 0.36249, + "4": 0.31969, + "5": 0.31782, + "6": 0.32168, + "7": 0.32353, + "8": 0.3154, + "9": 0.31087, + "10": 0.31551, + "11": 0.31395, + "12": 0.32071, + "13": 0.31429, + "14": 0.31261, + "15": 0.32212, + "16": 0.31397, + "17": 0.31669, + "18": 0.31709, + "19": 0.31473, + "20": 0.31387, + "21": 0.31467, + "22": 0.31261, + "23": 0.31257, + "24": 0.31319, + "25": 0.31222, + "26": 0.31451, + "27": 0.31085, + "28": 0.31197, + "29": 0.31593, + "30": 0.31292, + "31": 0.31046, + "32": 0.30787, + "33": 0.31043, + "34": 0.31354, + "35": 0.31328, + "36": 0.3168, + "37": 0.31378, + "38": 0.31603, + "39": 0.31204, + "40": 0.3124, + "41": 0.31379, + "42": 0.31443, + "43": 0.31174, + "44": 0.30994, + "45": 0.30904, + "46": 0.31159, + "47": 0.31115, + "48": 0.3143, + "49": 0.30675, + "50": 0.30906, + "51": 0.49669, + "52": 0.38931, + "53": 0.31571, + "54": 0.30992, + "55": 0.32391, + "56": 0.30868, + "57": 0.30741, + "58": 0.30841, + "59": 0.30986, + "60": 0.30856, + "61": 0.311, + "62": 0.30994, + "63": 0.30902, + "64": 0.31024, + "65": 0.31244, + "66": 0.31767, + "67": 0.31446, + "68": 0.3135, + "69": 0.31309, + "70": 0.31031, + "71": 0.309, + "72": 0.31049, + "73": 0.31242, + "74": 0.31631, + "75": 0.31327, + "76": 0.31167, + "77": 0.31945, + "78": 0.31913, + "79": 0.31484, + "80": 0.31853, + "81": 0.31285, + "82": 0.31341, + "83": 0.32958, + "84": 0.32372, + "85": 0.32705, + "86": 0.31969, + "87": 0.31329, + "88": 0.31888, + "89": 0.31874, + "90": 0.31709, + "91": 0.32147, + "92": 0.3233, + "93": 0.33785, + "94": 0.32293, + "95": 0.33598, + "96": 0.34111, + "97": 0.34363, + "98": 0.33613, + "99": 0.3257, + "100": 0.35727 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json index 025ee773139..7842d1c8574 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92624, - "2": 10.91761, - "3": 10.92333, - "4": 10.92688, - "5": 10.92998, - "6": 10.92516, + "1": 10.92561, + "2": 10.91856, + "3": 10.92257, + "4": 10.92726, + "5": 10.92964, + "6": 10.92362, "7": 10.9275, - "8": 10.91969, - "9": 10.92241, - "10": 10.92384, - "11": 10.90735, - "12": 10.92256, - "13": 10.90603, - "14": 10.88893, - "15": 10.89574, - "16": 10.87437, - "17": 10.88111, + "8": 10.9196, + "9": 10.92217, + "10": 10.92392, + "11": 10.90759, + "12": 10.92243, + "13": 10.90721, + "14": 10.88991, + "15": 10.89617, + "16": 10.87581, + "17": 10.87957, "18": 10.87989, - "19": 10.8777, - "20": 10.8229, - "21": 10.82151, - "22": 10.79833, - "23": 10.8093, - "24": 10.76998, - "25": 10.77846, - "26": 10.75625, - "27": 10.74877, - "28": 10.68549, - "29": 10.65912, - "30": 10.62682, - "31": 10.61174, - "32": 10.60594, - "33": 10.57889, - "34": 10.53573, - "35": 10.53734, - "36": 10.52861, - "37": 10.49704, - "38": 10.49542, - "39": 10.46092, - "40": 10.44217, - "41": 10.42129, - "42": 10.41516, - "43": 10.39816, - "44": 10.36695, - "45": 10.37226, - "46": 10.33554, - "47": 10.31809, - "48": 10.28656, - "49": 10.27539, - "50": 10.27508, - "51": 10.26992, - "52": 10.22163, - "53": 10.22906, - "54": 10.19419, - "55": 10.1737, - "56": 10.18831, - "57": 10.17869, - "58": 10.18534, - "59": 10.13498, - "60": 10.15562, - "61": 10.10616, - "62": 10.07447, - "63": 10.13218, - "64": 10.09215, - "65": 10.06779, - "66": 10.09487, - "67": 10.07265, - "68": 10.03581, - "69": 10.05526, - "70": 10.03316, - "71": 10.04956, - "72": 10.04089, - "73": 10.03831, - "74": 10.02118, - "75": 9.98217, - "76": 10.00359, - "77": 10.01464, - "78": 9.96825, - "79": 9.96968, - "80": 9.98497, - "81": 10.01098, - "82": 9.942, - "83": 9.91659, - "84": 9.85291, - "85": 9.83766, - "86": 9.94369, + "19": 10.87757, + "20": 10.82261, + "21": 10.82203, + "22": 10.79825, + "23": 10.80864, + "24": 10.76984, + "25": 10.77858, + "26": 10.756, + "27": 10.7495, + "28": 10.68555, + "29": 10.65925, + "30": 10.62653, + "31": 10.61181, + "32": 10.6054, + "33": 10.57914, + "34": 10.5363, + "35": 10.53794, + "36": 10.52773, + "37": 10.49731, + "38": 10.49549, + "39": 10.46025, + "40": 10.44235, + "41": 10.42131, + "42": 10.41507, + "43": 10.39818, + "44": 10.36665, + "45": 10.37237, + "46": 10.3366, + "47": 10.31802, + "48": 10.28685, + "49": 10.27543, + "50": 10.27528, + "51": 10.26998, + "52": 10.22134, + "53": 10.22978, + "54": 10.19405, + "55": 10.17419, + "56": 10.18856, + "57": 10.17853, + "58": 10.18551, + "59": 10.13513, + "60": 10.15563, + "61": 10.10614, + "62": 10.07409, + "63": 10.13178, + "64": 10.09178, + "65": 10.06805, + "66": 10.09521, + "67": 10.0727, + "68": 10.03552, + "69": 10.05548, + "70": 10.03291, + "71": 10.04966, + "72": 10.04135, + "73": 10.03852, + "74": 10.0208, + "75": 9.98234, + "76": 10.00344, + "77": 10.01482, + "78": 9.96829, + "79": 9.96986, + "80": 9.98503, + "81": 10.01099, + "82": 9.94184, + "83": 9.91679, + "84": 9.85314, + "85": 9.83762, + "86": 9.94372, "87": 9.95599, - "88": 9.93523, - "89": 9.87132, - "90": 9.8789, - "91": 9.89514, - "92": 9.86906, - "93": 9.8012, - "94": 9.8889, - "95": 9.86015, - "96": 9.849, - "97": 9.79912, - "98": 9.83254, - "99": 9.8645, - "100": 9.76002 + "88": 9.93505, + "89": 9.87147, + "90": 9.87922, + "91": 9.89549, + "92": 9.86949, + "93": 9.80192, + "94": 9.88899, + "95": 9.86036, + "96": 9.84943, + "97": 9.79943, + "98": 9.83227, + "99": 9.86459, + "100": 9.76036 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36362.0, - "2": 36609.0, - "3": 36699.0, - "4": 36555.0, - "5": 36453.0, - "6": 36089.0, - "7": 37058.0, - "8": 36230.0, - "9": 37141.0, - "10": 37547.0, - "11": 36627.0, - "12": 35253.0, - "13": 36871.0, - "14": 36920.0, - "15": 36125.0, - "16": 35933.0, - "17": 36498.0, - "18": 36647.0, - "19": 36906.0, - "20": 36544.0, - "21": 36047.0, - "22": 36411.0, - "23": 36227.0, - "24": 36399.0, - "25": 36161.0, - "26": 36511.0, - "27": 36607.0, - "28": 35615.0, - "29": 35641.0, - "30": 35308.0, - "31": 35928.0, - "32": 36626.0, - "33": 35309.0, - "34": 35493.0, - "35": 36313.0, - "36": 35348.0, - "37": 35771.0, - "38": 35727.0, - "39": 36751.0, - "40": 37556.0, - "41": 36096.0, - "42": 35169.0, - "43": 37377.0, - "44": 36413.0, - "45": 39294.0, - "46": 38016.0, - "47": 38125.0, - "48": 38571.0, - "49": 41964.0, - "50": 38513.0, - "51": 38718.0, - "52": 42043.0, - "53": 40110.0, - "54": 41376.0, - "55": 38983.0, - "56": 40743.0, - "57": 37045.0, - "58": 45842.0, - "59": 41774.0, - "60": 42246.0, - "61": 41868.0, - "62": 44805.0, - "63": 43800.0, - "64": 49706.0, - "65": 41660.0, - "66": 45388.0, - "67": 50065.0, - "68": 47387.0, - "69": 44392.0, - "70": 47481.0, - "71": 45682.0, - "72": 45161.0, - "73": 48046.0, - "74": 45712.0, - "75": 47057.0, - "76": 46874.0, - "77": 47480.0, - "78": 49142.0, - "79": 46944.0, - "80": 45550.0, - "81": 54133.0, - "82": 41978.0, - "83": 46216.0, - "84": 45475.0, - "85": 45262.0, - "86": 45607.0, - "87": 37027.0, - "88": 46017.0, - "89": 48872.0, - "90": 56923.0, - "91": 38456.0, - "92": 49398.0, - "93": 46732.0, - "94": 42779.0, - "95": 45860.0, - "96": 50249.0, - "97": 47569.0, - "98": 49525.0, - "99": 44279.0, - "100": 45708.0 + "1": 36436.0, + "2": 36501.0, + "3": 36802.0, + "4": 36766.0, + "5": 36386.0, + "6": 35733.0, + "7": 36883.0, + "8": 35982.0, + "9": 37134.0, + "10": 37619.0, + "11": 36856.0, + "12": 35393.0, + "13": 36987.0, + "14": 36916.0, + "15": 36126.0, + "16": 36007.0, + "17": 36166.0, + "18": 36642.0, + "19": 36691.0, + "20": 36267.0, + "21": 36325.0, + "22": 36849.0, + "23": 36559.0, + "24": 36807.0, + "25": 35927.0, + "26": 36377.0, + "27": 36395.0, + "28": 35441.0, + "29": 35838.0, + "30": 35285.0, + "31": 35827.0, + "32": 36688.0, + "33": 35140.0, + "34": 35614.0, + "35": 36283.0, + "36": 35123.0, + "37": 36078.0, + "38": 35203.0, + "39": 36835.0, + "40": 37651.0, + "41": 35925.0, + "42": 35887.0, + "43": 37261.0, + "44": 36196.0, + "45": 39547.0, + "46": 38230.0, + "47": 37972.0, + "48": 38405.0, + "49": 41846.0, + "50": 38991.0, + "51": 38781.0, + "52": 41557.0, + "53": 40236.0, + "54": 41428.0, + "55": 39279.0, + "56": 40896.0, + "57": 36831.0, + "58": 46014.0, + "59": 41870.0, + "60": 42002.0, + "61": 41281.0, + "62": 44376.0, + "63": 43888.0, + "64": 50350.0, + "65": 41693.0, + "66": 45434.0, + "67": 49882.0, + "68": 47180.0, + "69": 44903.0, + "70": 47813.0, + "71": 45797.0, + "72": 45119.0, + "73": 48038.0, + "74": 46007.0, + "75": 46998.0, + "76": 47058.0, + "77": 47604.0, + "78": 49536.0, + "79": 46549.0, + "80": 45509.0, + "81": 53807.0, + "82": 42629.0, + "83": 46031.0, + "84": 45537.0, + "85": 45231.0, + "86": 44892.0, + "87": 37088.0, + "88": 46123.0, + "89": 48687.0, + "90": 57077.0, + "91": 38387.0, + "92": 49670.0, + "93": 46990.0, + "94": 42760.0, + "95": 46020.0, + "96": 50224.0, + "97": 47737.0, + "98": 49761.0, + "99": 44729.0, + "100": 45627.0 } }, "mem-allocated-bytes": { @@ -218,105 +218,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254751744.0, - "2": 1254746624.0, - "3": 1254748160.0, - "4": 1254748160.0, + "1": 1254750720.0, + "2": 1254748160.0, + "3": 1254747136.0, + "4": 1254748672.0, "5": 1254748672.0, - "6": 1254751744.0, - "7": 1254750208.0, - "8": 1254747648.0, - "9": 1254748672.0, - "10": 1254747648.0, + "6": 1254752256.0, + "7": 1254750720.0, + "8": 1254747136.0, + "9": 1254747136.0, + "10": 1254746624.0, "11": 1254749184.0, - "12": 1254747136.0, - "13": 1254750720.0, + "12": 1254746624.0, + "13": 1254749696.0, "14": 1254748672.0, - "15": 1254748672.0, - "16": 1254746112.0, - "17": 1254747136.0, + "15": 1254747648.0, + "16": 1254747648.0, + "17": 1254748160.0, "18": 1254746624.0, - "19": 1254752256.0, + "19": 1254751744.0, "20": 1254746112.0, - "21": 1254748160.0, - "22": 1254749184.0, - "23": 1254749696.0, - "24": 1254747648.0, + "21": 1254747648.0, + "22": 1254748672.0, + "23": 1254749184.0, + "24": 1254747136.0, "25": 1254748160.0, - "26": 1254745600.0, + "26": 1254746624.0, "27": 1254749696.0, "28": 1254748160.0, - "29": 1254752768.0, - "30": 1254747648.0, - "31": 1254749184.0, - "32": 1254746112.0, + "29": 1254751744.0, + "30": 1254747136.0, + "31": 1254748160.0, + "32": 1254746624.0, "33": 1254748160.0, - "34": 1254747136.0, - "35": 1254746112.0, - "36": 1254748672.0, + "34": 1254745600.0, + "35": 1254747136.0, + "36": 1254748160.0, "37": 1254748160.0, - "38": 1254747648.0, - "39": 1254747136.0, - "40": 1254750720.0, - "41": 1254748160.0, - "42": 1254748672.0, - "43": 1254746624.0, - "44": 1254748160.0, - "45": 1254748160.0, - "46": 1254749184.0, - "47": 1254748672.0, - "48": 1254747648.0, - "49": 1254747648.0, - "50": 1254749696.0, - "51": 1254746624.0, - "52": 1254747648.0, - "53": 1254747136.0, + "38": 1254746624.0, + "39": 1254748160.0, + "40": 1254752256.0, + "41": 1254748672.0, + "42": 1254748160.0, + "43": 1254746112.0, + "44": 1254747648.0, + "45": 1254749696.0, + "46": 1254749696.0, + "47": 1254749696.0, + "48": 1254748160.0, + "49": 1254746624.0, + "50": 1254748672.0, + "51": 1254746112.0, + "52": 1254746624.0, + "53": 1254746624.0, "54": 1254747648.0, "55": 1254749184.0, - "56": 1254747136.0, + "56": 1254747648.0, "57": 1254749184.0, - "58": 1254749696.0, - "59": 1254747136.0, - "60": 1254745600.0, + "58": 1254749184.0, + "59": 1254746624.0, + "60": 1254745088.0, "61": 1254747136.0, - "62": 1254746624.0, - "63": 1254747136.0, + "62": 1254747136.0, + "63": 1254746112.0, "64": 1254748160.0, - "65": 1254747136.0, - "66": 1254750208.0, - "67": 1254745600.0, - "68": 1254749184.0, - "69": 1254746112.0, + "65": 1254747648.0, + "66": 1254749184.0, + "67": 1254746624.0, + "68": 1254750208.0, + "69": 1254745088.0, "70": 1254746112.0, - "71": 1254747136.0, + "71": 1254748160.0, "72": 1254750720.0, - "73": 1254746624.0, - "74": 1254746112.0, - "75": 1254749184.0, - "76": 1254748160.0, + "73": 1254747136.0, + "74": 1254745600.0, + "75": 1254748672.0, + "76": 1254747648.0, "77": 1254741504.0, - "78": 1254745600.0, - "79": 1254746112.0, + "78": 1254746112.0, + "79": 1254747648.0, "80": 1254745600.0, "81": 1254742528.0, - "82": 1254744064.0, - "83": 1254747136.0, - "84": 1254742016.0, - "85": 1254739968.0, - "86": 1254732288.0, - "87": 1254746112.0, - "88": 1254750208.0, - "89": 1254739456.0, + "82": 1254743040.0, + "83": 1254746624.0, + "84": 1254743040.0, + "85": 1254741504.0, + "86": 1254733312.0, + "87": 1254746624.0, + "88": 1254748160.0, + "89": 1254737920.0, "90": 1254739968.0, - "91": 1254746624.0, - "92": 1254736384.0, - "93": 1254741504.0, - "94": 1254736384.0, - "95": 1254732800.0, - "96": 1254748672.0, - "97": 1254734848.0, + "91": 1254747136.0, + "92": 1254735360.0, + "93": 1254742016.0, + "94": 1254737408.0, + "95": 1254733824.0, + "96": 1254747648.0, + "97": 1254734336.0, "98": 1254728192.0, - "99": 1254733312.0, + "99": 1254734848.0, "100": 1254732800.0 } }, @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2042493440.0, - "2": 2517739520.0, - "3": 2518202368.0, - "4": 2518202368.0, - "5": 2518202368.0, - "6": 2519861760.0, - "7": 2519861760.0, - "8": 2519861760.0, - "9": 2519861760.0, - "10": 2519861760.0, - "11": 2519861760.0, - "12": 2519861760.0, - "13": 2519861760.0, - "14": 2519861760.0, - "15": 2519861760.0, - "16": 2519861760.0, - "17": 2519861760.0, - "18": 2519861760.0, - "19": 2520205312.0, - "20": 2520205312.0, - "21": 2520205312.0, - "22": 2520205312.0, - "23": 2520205312.0, - "24": 2520205312.0, - "25": 2520205312.0, - "26": 2520205312.0, - "27": 2520205312.0, - "28": 2520205312.0, - "29": 2520205312.0, - "30": 2520205312.0, - "31": 2520205312.0, - "32": 2520205312.0, - "33": 2520205312.0, - "34": 2520205312.0, - "35": 2520205312.0, - "36": 2520205312.0, - "37": 2520205312.0, - "38": 2520205312.0, - "39": 2520205312.0, - "40": 2520205312.0, - "41": 2520205312.0, - "42": 2520205312.0, - "43": 2520205312.0, - "44": 2520205312.0, - "45": 2520205312.0, - "46": 2520205312.0, - "47": 2520205312.0, - "48": 2520205312.0, - "49": 2520205312.0, - "50": 2520205312.0, - "51": 2520205312.0, - "52": 2520205312.0, - "53": 2520205312.0, - "54": 2520205312.0, - "55": 2520205312.0, - "56": 2520205312.0, - "57": 2520205312.0, - "58": 2520205312.0, - "59": 2520205312.0, - "60": 2520205312.0, - "61": 2520205312.0, - "62": 2520205312.0, - "63": 2520205312.0, - "64": 2520205312.0, - "65": 2520205312.0, - "66": 2520205312.0, - "67": 2520205312.0, - "68": 2520205312.0, - "69": 2520205312.0, - "70": 2520205312.0, - "71": 2520205312.0, - "72": 2520205312.0, - "73": 2520205312.0, - "74": 2520205312.0, - "75": 2520205312.0, - "76": 2520205312.0, - "77": 2520205312.0, - "78": 2520205312.0, - "79": 2520205312.0, - "80": 2520205312.0, - "81": 2520205312.0, - "82": 2520205312.0, - "83": 2520205312.0, - "84": 2520205312.0, - "85": 2520205312.0, - "86": 2520205312.0, - "87": 2520205312.0, - "88": 2520205312.0, - "89": 2520205312.0, - "90": 2520205312.0, - "91": 2520205312.0, - "92": 2520205312.0, - "93": 2520205312.0, - "94": 2520205312.0, - "95": 2520205312.0, - "96": 2520205312.0, - "97": 2520205312.0, - "98": 2520205312.0, - "99": 2520205312.0, - "100": 2520205312.0 + "1": 2042772992.0, + "2": 2517462016.0, + "3": 2518086656.0, + "4": 2518086656.0, + "5": 2518086656.0, + "6": 2520246272.0, + "7": 2520246272.0, + "8": 2520246272.0, + "9": 2520246272.0, + "10": 2520246272.0, + "11": 2520246272.0, + "12": 2520246272.0, + "13": 2520246272.0, + "14": 2520246272.0, + "15": 2520246272.0, + "16": 2520246272.0, + "17": 2520246272.0, + "18": 2520246272.0, + "19": 2520246272.0, + "20": 2520246272.0, + "21": 2520246272.0, + "22": 2520246272.0, + "23": 2520246272.0, + "24": 2520246272.0, + "25": 2520246272.0, + "26": 2520246272.0, + "27": 2520246272.0, + "28": 2520246272.0, + "29": 2520246272.0, + "30": 2520246272.0, + "31": 2520246272.0, + "32": 2520246272.0, + "33": 2520246272.0, + "34": 2520246272.0, + "35": 2520246272.0, + "36": 2520246272.0, + "37": 2520246272.0, + "38": 2520246272.0, + "39": 2520246272.0, + "40": 2520246272.0, + "41": 2520246272.0, + "42": 2520246272.0, + "43": 2520246272.0, + "44": 2520246272.0, + "45": 2520246272.0, + "46": 2520246272.0, + "47": 2520246272.0, + "48": 2520246272.0, + "49": 2520246272.0, + "50": 2520246272.0, + "51": 2520246272.0, + "52": 2520246272.0, + "53": 2520246272.0, + "54": 2520246272.0, + "55": 2520246272.0, + "56": 2520246272.0, + "57": 2520246272.0, + "58": 2520246272.0, + "59": 2520246272.0, + "60": 2520246272.0, + "61": 2520246272.0, + "62": 2520246272.0, + "63": 2520246272.0, + "64": 2520246272.0, + "65": 2520246272.0, + "66": 2520246272.0, + "67": 2520246272.0, + "68": 2520246272.0, + "69": 2520246272.0, + "70": 2520246272.0, + "71": 2520246272.0, + "72": 2520246272.0, + "73": 2520246272.0, + "74": 2520246272.0, + "75": 2520246272.0, + "76": 2520246272.0, + "77": 2520246272.0, + "78": 2520246272.0, + "79": 2520246272.0, + "80": 2520246272.0, + "81": 2520246272.0, + "82": 2520246272.0, + "83": 2520246272.0, + "84": 2520246272.0, + "85": 2520246272.0, + "86": 2520246272.0, + "87": 2520246272.0, + "88": 2520246272.0, + "89": 2520246272.0, + "90": 2520246272.0, + "91": 2520246272.0, + "92": 2520246272.0, + "93": 2520246272.0, + "94": 2520246272.0, + "95": 2520246272.0, + "96": 2520246272.0, + "97": 2520246272.0, + "98": 2520246272.0, + "99": 2520246272.0, + "100": 2520246272.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.6311, - "3": 0.34592, - "4": 0.34521, - "5": 0.32602, - "6": 0.32592, - "7": 0.33225, - "8": 0.32209, - "9": 0.31799, - "10": 0.32175, - "11": 0.31963, - "12": 0.31948, - "13": 0.32628, - "14": 0.32525, - "15": 0.32035, - "16": 0.31631, - "17": 0.3172, - "18": 0.31752, - "19": 0.31362, - "20": 0.3134, - "21": 0.3139, - "22": 0.31635, - "23": 0.31613, - "24": 0.31472, - "25": 0.31973, - "26": 0.31472, - "27": 0.31887, - "28": 0.31268, - "29": 0.31511, - "30": 0.31604, - "31": 0.31415, - "32": 0.31381, - "33": 0.31328, - "34": 0.3143, - "35": 0.3122, - "36": 0.30943, - "37": 0.31565, - "38": 0.31793, - "39": 0.32043, - "40": 0.32101, - "41": 0.31783, - "42": 0.31527, - "43": 0.31523, - "44": 0.31606, - "45": 0.31171, - "46": 0.31406, - "47": 0.31657, - "48": 0.32111, - "49": 0.31335, - "50": 0.32346, - "51": 0.37683, - "52": 0.38404, - "53": 0.32399, - "54": 0.32106, - "55": 0.32305, - "56": 0.31666, - "57": 0.31622, - "58": 0.32015, - "59": 0.31487, - "60": 0.32232, - "61": 0.3198, - "62": 0.31879, - "63": 0.31919, - "64": 0.86198, - "65": 0.32672, - "66": 0.31588, - "67": 0.31858, - "68": 0.32169, - "69": 0.31983, - "70": 0.31958, - "71": 0.32528, - "72": 0.3201, - "73": 0.32347, - "74": 0.33992, - "75": 0.33753, - "76": 0.32343, - "77": 0.34573, - "78": 0.33165, - "79": 0.32229, - "80": 0.33411, - "81": 0.32778, - "82": 0.33225, - "83": 0.36857, - "84": 0.34612, - "85": 0.34462, - "86": 0.34803, - "87": 0.32667, - "88": 0.32339, - "89": 0.35636, - "90": 0.33681, - "91": 0.3477, - "92": 0.33318, - "93": 0.35303, - "94": 0.33954, - "95": 0.3624, - "96": 0.38791, - "97": 0.36171, - "98": 0.35126, - "99": 0.33264, - "100": 0.34722 + "2": 6.20285, + "3": 0.34784, + "4": 0.34202, + "5": 0.34201, + "6": 0.33485, + "7": 0.33457, + "8": 0.34157, + "9": 0.32938, + "10": 0.33046, + "11": 0.3269, + "12": 0.33179, + "13": 0.32939, + "14": 0.32919, + "15": 0.32852, + "16": 0.32762, + "17": 0.32328, + "18": 0.32911, + "19": 0.32343, + "20": 0.32629, + "21": 0.3267, + "22": 0.33068, + "23": 0.33086, + "24": 0.32953, + "25": 0.33198, + "26": 0.32657, + "27": 0.33229, + "28": 0.32408, + "29": 0.33113, + "30": 0.32566, + "31": 0.3257, + "32": 0.32274, + "33": 0.32121, + "34": 0.32519, + "35": 0.32311, + "36": 0.32414, + "37": 0.32891, + "38": 0.32308, + "39": 0.32046, + "40": 0.32292, + "41": 0.32324, + "42": 0.32332, + "43": 0.3222, + "44": 0.33622, + "45": 0.32352, + "46": 0.32214, + "47": 0.32315, + "48": 0.33134, + "49": 0.32378, + "50": 0.32327, + "51": 0.34347, + "52": 0.37235, + "53": 0.3301, + "54": 0.33158, + "55": 0.32431, + "56": 0.32532, + "57": 0.32045, + "58": 0.32757, + "59": 0.32448, + "60": 0.32369, + "61": 0.33182, + "62": 0.32329, + "63": 0.32144, + "64": 0.3216, + "65": 0.31962, + "66": 0.31986, + "67": 0.32382, + "68": 0.32461, + "69": 0.33864, + "70": 0.32369, + "71": 0.3281, + "72": 0.32125, + "73": 0.32568, + "74": 0.33699, + "75": 0.33558, + "76": 0.32825, + "77": 0.33583, + "78": 0.34024, + "79": 0.91474, + "80": 0.32921, + "81": 0.32621, + "82": 0.32729, + "83": 0.38345, + "84": 0.34586, + "85": 0.34868, + "86": 0.43193, + "87": 0.32154, + "88": 0.33733, + "89": 0.42244, + "90": 0.34665, + "91": 0.33835, + "92": 0.33996, + "93": 0.4351, + "94": 1.12125, + "95": 0.42067, + "96": 0.38353, + "97": 0.3737, + "98": 0.36165, + "99": 0.35383, + "100": 0.40097 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json index fe59d4e351b..904dedf2418 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92456, - "2": 10.91122, - "3": 10.92478, - "4": 10.91392, - "5": 10.91847, - "6": 10.90864, - "7": 10.90181, - "8": 10.90918, - "9": 10.92285, - "10": 10.91171, - "11": 10.90188, - "12": 10.91191, - "13": 10.88926, - "14": 10.8908, - "15": 10.88438, - "16": 10.87546, - "17": 10.87471, - "18": 10.8628, - "19": 10.8724, - "20": 10.8201, - "21": 10.8072, - "22": 10.79503, - "23": 10.79079, - "24": 10.75741, - "25": 10.76214, - "26": 10.75151, - "27": 10.73656, - "28": 10.67545, - "29": 10.63996, - "30": 10.62263, - "31": 10.61218, - "32": 10.59655, - "33": 10.5765, - "34": 10.53881, - "35": 10.53788, - "36": 10.5216, - "37": 10.48667, - "38": 10.49899, - "39": 10.45883, - "40": 10.44007, - "41": 10.42695, - "42": 10.41344, - "43": 10.38989, - "44": 10.35254, - "45": 10.36981, - "46": 10.33007, - "47": 10.31289, - "48": 10.27798, - "49": 10.27103, - "50": 10.27187 + "1": 10.92337, + "2": 10.91184, + "3": 10.9245, + "4": 10.91413, + "5": 10.91791, + "6": 10.90788, + "7": 10.90285, + "8": 10.9094, + "9": 10.92264, + "10": 10.91325, + "11": 10.90217, + "12": 10.91139, + "13": 10.88971, + "14": 10.88954, + "15": 10.88501, + "16": 10.87598, + "17": 10.87426, + "18": 10.8634, + "19": 10.87212, + "20": 10.82076, + "21": 10.80748, + "22": 10.79544, + "23": 10.79148, + "24": 10.75643, + "25": 10.76142, + "26": 10.75154, + "27": 10.73598, + "28": 10.67469, + "29": 10.64016, + "30": 10.62245, + "31": 10.61276, + "32": 10.59683, + "33": 10.57641, + "34": 10.53937, + "35": 10.53846, + "36": 10.52255, + "37": 10.48675, + "38": 10.49927, + "39": 10.45887, + "40": 10.43983, + "41": 10.42774, + "42": 10.41415, + "43": 10.3903, + "44": 10.35238, + "45": 10.36987, + "46": 10.33008, + "47": 10.31244, + "48": 10.27782, + "49": 10.27035, + "50": 10.27145 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18865.0, - "2": 18932.0, - "3": 18832.0, - "4": 18774.0, - "5": 18548.0, - "6": 18607.0, - "7": 18950.0, - "8": 18828.0, - "9": 19101.0, - "10": 19425.0, - "11": 19225.0, - "12": 18243.0, - "13": 19160.0, - "14": 19145.0, - "15": 18772.0, - "16": 18768.0, - "17": 18807.0, - "18": 19111.0, - "19": 18890.0, - "20": 19015.0, - "21": 18479.0, - "22": 18922.0, - "23": 18880.0, - "24": 18812.0, - "25": 18185.0, - "26": 18922.0, - "27": 18466.0, - "28": 18402.0, - "29": 18510.0, - "30": 18253.0, - "31": 18896.0, - "32": 18852.0, - "33": 18338.0, - "34": 18553.0, - "35": 18775.0, - "36": 18344.0, - "37": 19084.0, - "38": 18477.0, - "39": 18845.0, - "40": 19477.0, - "41": 18606.0, - "42": 18524.0, - "43": 19412.0, - "44": 18862.0, - "45": 20342.0, - "46": 19670.0, - "47": 19622.0, - "48": 19999.0, - "49": 21560.0, - "50": 20111.0 + "1": 18811.0, + "2": 19165.0, + "3": 19169.0, + "4": 18794.0, + "5": 18806.0, + "6": 18547.0, + "7": 18947.0, + "8": 18963.0, + "9": 18914.0, + "10": 19344.0, + "11": 18881.0, + "12": 18329.0, + "13": 18937.0, + "14": 18987.0, + "15": 18708.0, + "16": 18957.0, + "17": 19271.0, + "18": 18855.0, + "19": 18843.0, + "20": 19091.0, + "21": 18628.0, + "22": 18905.0, + "23": 18905.0, + "24": 19000.0, + "25": 18306.0, + "26": 18509.0, + "27": 18924.0, + "28": 18259.0, + "29": 18452.0, + "30": 18270.0, + "31": 18685.0, + "32": 18898.0, + "33": 18250.0, + "34": 18394.0, + "35": 18736.0, + "36": 18582.0, + "37": 18806.0, + "38": 18396.0, + "39": 19248.0, + "40": 19332.0, + "41": 18568.0, + "42": 18144.0, + "43": 19243.0, + "44": 18642.0, + "45": 20136.0, + "46": 19403.0, + "47": 19891.0, + "48": 20076.0, + "49": 21584.0, + "50": 20152.0 } }, "mem-allocated-bytes": { @@ -118,55 +118,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388730880.0, - "2": 1388731392.0, + "1": 1388729856.0, + "2": 1388730368.0, "3": 1388729856.0, - "4": 1388727296.0, - "5": 1388729856.0, + "4": 1388727808.0, + "5": 1388728832.0, "6": 1388730368.0, - "7": 1388729856.0, + "7": 1388731904.0, "8": 1388729856.0, - "9": 1388731392.0, + "9": 1388731904.0, "10": 1388731392.0, - "11": 1388729344.0, + "11": 1388728832.0, "12": 1388728832.0, "13": 1388730880.0, "14": 1388728320.0, - "15": 1388730880.0, - "16": 1388729344.0, - "17": 1388731904.0, - "18": 1388730880.0, - "19": 1388730368.0, + "15": 1388731904.0, + "16": 1388728320.0, + "17": 1388730880.0, + "18": 1388730368.0, + "19": 1388729856.0, "20": 1388730368.0, - "21": 1388731392.0, - "22": 1388730880.0, + "21": 1388732928.0, + "22": 1388729344.0, "23": 1388731392.0, - "24": 1388730880.0, - "25": 1388731904.0, + "24": 1388729344.0, + "25": 1388732928.0, "26": 1388732928.0, - "27": 1388731392.0, - "28": 1388729856.0, + "27": 1388729344.0, + "28": 1388730368.0, "29": 1388731392.0, "30": 1388733440.0, "31": 1388732416.0, - "32": 1388730368.0, + "32": 1388729344.0, "33": 1388732416.0, "34": 1388734464.0, - "35": 1388731904.0, + "35": 1388731392.0, "36": 1388730368.0, "37": 1388731904.0, - "38": 1388729856.0, - "39": 1388732928.0, - "40": 1388733952.0, - "41": 1388729856.0, + "38": 1388730368.0, + "39": 1388733440.0, + "40": 1388734464.0, + "41": 1388730368.0, "42": 1388732928.0, "43": 1388731392.0, "44": 1388729856.0, - "45": 1388733952.0, - "46": 1388731392.0, - "47": 1388732928.0, + "45": 1388732928.0, + "46": 1388731904.0, + "47": 1388733952.0, "48": 1388731904.0, - "49": 1388736000.0, + "49": 1388736512.0, "50": 1388731904.0 } }, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179614208.0, - "2": 3661343744.0, - "3": 3663287296.0, - "4": 3663287296.0, - "5": 3663287296.0, - "6": 3663287296.0, - "7": 3663287296.0, - "8": 3663287296.0, - "9": 3663287296.0, - "10": 3663287296.0, - "11": 3663287296.0, - "12": 3663287296.0, - "13": 3663287296.0, - "14": 3663287296.0, - "15": 3663287296.0, - "16": 3663287296.0, - "17": 3663287296.0, - "18": 3663287296.0, - "19": 3663287296.0, - "20": 3663287296.0, - "21": 3663287296.0, - "22": 3663287296.0, - "23": 3663287296.0, - "24": 3663287296.0, - "25": 3663287296.0, - "26": 3663287296.0, - "27": 3663287296.0, - "28": 3663287296.0, - "29": 3663287296.0, - "30": 3663287296.0, - "31": 3663287296.0, - "32": 3663287296.0, - "33": 3663287296.0, - "34": 3664704512.0, - "35": 3664704512.0, - "36": 3664704512.0, - "37": 3664704512.0, - "38": 3664704512.0, - "39": 3664704512.0, - "40": 3664704512.0, - "41": 3664704512.0, - "42": 3664704512.0, - "43": 3664704512.0, - "44": 3664704512.0, - "45": 3664704512.0, - "46": 3665018368.0, - "47": 3665018368.0, - "48": 3665595904.0, - "49": 3665595904.0, - "50": 3665595904.0 + "1": 3179474432.0, + "2": 3662153216.0, + "3": 3662153216.0, + "4": 3662153216.0, + "5": 3662153216.0, + "6": 3662153216.0, + "7": 3662153216.0, + "8": 3662153216.0, + "9": 3662153216.0, + "10": 3662261248.0, + "11": 3662261248.0, + "12": 3662261248.0, + "13": 3662261248.0, + "14": 3662261248.0, + "15": 3662261248.0, + "16": 3662261248.0, + "17": 3662261248.0, + "18": 3662261248.0, + "19": 3662745600.0, + "20": 3662745600.0, + "21": 3662745600.0, + "22": 3662745600.0, + "23": 3662745600.0, + "24": 3662745600.0, + "25": 3662901760.0, + "26": 3663353344.0, + "27": 3663353344.0, + "28": 3663353344.0, + "29": 3663353344.0, + "30": 3663353344.0, + "31": 3663353344.0, + "32": 3663353344.0, + "33": 3663353344.0, + "34": 3663353344.0, + "35": 3663353344.0, + "36": 3663353344.0, + "37": 3663353344.0, + "38": 3663353344.0, + "39": 3664240640.0, + "40": 3664613376.0, + "41": 3664613376.0, + "42": 3664613376.0, + "43": 3664613376.0, + "44": 3664749056.0, + "45": 3664749056.0, + "46": 3664804352.0, + "47": 3664804352.0, + "48": 3664804352.0, + "49": 3664804352.0, + "50": 3664804352.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.32018, - "3": 1.74495, - "4": 1.64524, - "5": 0.85018, - "6": 1.50819, - "7": 1.44545, - "8": 1.66946, - "9": 1.72718, - "10": 1.18171, - "11": 1.35307, - "12": 1.46593, - "13": 1.24052, - "14": 1.87146, - "15": 0.99226, - "16": 1.29853, - "17": 1.35074, - "18": 1.36129, - "19": 1.39156, - "20": 1.72658, - "21": 1.32313, - "22": 1.72423, - "23": 1.72372, - "24": 2.64381, - "25": 1.46675, - "26": 2.26793, - "27": 1.80467, - "28": 3.05207, - "29": 2.34622, - "30": 1.46261, - "31": 2.24908, - "32": 2.42053, - "33": 1.30876, - "34": 1.44075, - "35": 2.05636, - "36": 1.56688, - "37": 1.6821, - "38": 1.48888, - "39": 1.57894, - "40": 1.92345, - "41": 1.46836, - "42": 2.77498, - "43": 1.72973, - "44": 2.152, - "45": 0.78685, - "46": 1.53871, - "47": 2.04434, - "48": 1.7392, - "49": 1.41271, - "50": 2.00804 + "2": 5.16405, + "3": 1.57287, + "4": 1.26821, + "5": 0.79832, + "6": 1.34333, + "7": 1.25522, + "8": 1.54632, + "9": 1.61544, + "10": 1.02137, + "11": 1.26697, + "12": 1.17489, + "13": 1.04062, + "14": 1.46403, + "15": 0.84633, + "16": 1.0962, + "17": 1.28286, + "18": 1.25619, + "19": 1.35218, + "20": 1.27334, + "21": 1.23896, + "22": 1.17865, + "23": 1.31627, + "24": 1.82184, + "25": 0.7204, + "26": 1.02471, + "27": 1.06414, + "28": 1.13104, + "29": 0.92276, + "30": 1.05126, + "31": 0.92313, + "32": 1.9269, + "33": 1.13456, + "34": 0.58809, + "35": 1.4753, + "36": 1.42283, + "37": 1.1591, + "38": 0.67365, + "39": 1.39463, + "40": 1.68848, + "41": 1.17903, + "42": 1.39721, + "43": 1.27706, + "44": 1.55428, + "45": 0.66041, + "46": 1.8717, + "47": 1.40133, + "48": 1.31439, + "49": 0.8202, + "50": 1.42077 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json index fbaf488494a..e1068fd36e9 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92456, - "2": 10.91122, - "3": 10.92478, - "4": 10.91392, - "5": 10.91847, - "6": 10.90864, - "7": 10.90181, - "8": 10.90918, - "9": 10.92285, - "10": 10.91171, - "11": 10.90188, - "12": 10.91191, - "13": 10.88926, - "14": 10.8908, - "15": 10.88438, - "16": 10.87546, - "17": 10.87471, - "18": 10.8628, - "19": 10.8724, - "20": 10.8201, - "21": 10.8072, - "22": 10.79503, - "23": 10.79079, - "24": 10.75741, - "25": 10.76214, - "26": 10.75151, - "27": 10.73656, - "28": 10.67545, - "29": 10.63996, - "30": 10.62263, - "31": 10.61218, - "32": 10.59655, - "33": 10.5765, - "34": 10.53881, - "35": 10.53788, - "36": 10.5216, - "37": 10.48667, - "38": 10.49899, - "39": 10.45883, - "40": 10.44007, - "41": 10.42695, - "42": 10.41344, - "43": 10.38989, - "44": 10.35254, - "45": 10.36981, - "46": 10.33007, - "47": 10.31289, - "48": 10.27798, - "49": 10.27103, - "50": 10.27187 + "1": 10.92337, + "2": 10.91184, + "3": 10.9245, + "4": 10.91413, + "5": 10.91791, + "6": 10.90788, + "7": 10.90285, + "8": 10.9094, + "9": 10.92264, + "10": 10.91325, + "11": 10.90217, + "12": 10.91139, + "13": 10.88971, + "14": 10.88954, + "15": 10.88501, + "16": 10.87598, + "17": 10.87426, + "18": 10.8634, + "19": 10.87212, + "20": 10.82076, + "21": 10.80748, + "22": 10.79544, + "23": 10.79148, + "24": 10.75643, + "25": 10.76142, + "26": 10.75154, + "27": 10.73598, + "28": 10.67469, + "29": 10.64016, + "30": 10.62245, + "31": 10.61276, + "32": 10.59683, + "33": 10.57641, + "34": 10.53937, + "35": 10.53846, + "36": 10.52255, + "37": 10.48675, + "38": 10.49927, + "39": 10.45887, + "40": 10.43983, + "41": 10.42774, + "42": 10.41415, + "43": 10.3903, + "44": 10.35238, + "45": 10.36987, + "46": 10.33008, + "47": 10.31244, + "48": 10.27782, + "49": 10.27035, + "50": 10.27145 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18865.0, - "2": 18932.0, - "3": 18832.0, - "4": 18774.0, - "5": 18548.0, - "6": 18607.0, - "7": 18950.0, - "8": 18828.0, - "9": 19101.0, - "10": 19425.0, - "11": 19225.0, - "12": 18243.0, - "13": 19160.0, - "14": 19145.0, - "15": 18772.0, - "16": 18768.0, - "17": 18807.0, - "18": 19111.0, - "19": 18890.0, - "20": 19015.0, - "21": 18479.0, - "22": 18922.0, - "23": 18880.0, - "24": 18812.0, - "25": 18185.0, - "26": 18922.0, - "27": 18466.0, - "28": 18402.0, - "29": 18510.0, - "30": 18253.0, - "31": 18896.0, - "32": 18852.0, - "33": 18338.0, - "34": 18553.0, - "35": 18775.0, - "36": 18344.0, - "37": 19084.0, - "38": 18477.0, - "39": 18845.0, - "40": 19477.0, - "41": 18606.0, - "42": 18524.0, - "43": 19412.0, - "44": 18862.0, - "45": 20342.0, - "46": 19670.0, - "47": 19622.0, - "48": 19999.0, - "49": 21560.0, - "50": 20111.0 + "1": 18811.0, + "2": 19165.0, + "3": 19169.0, + "4": 18794.0, + "5": 18806.0, + "6": 18547.0, + "7": 18947.0, + "8": 18963.0, + "9": 18914.0, + "10": 19344.0, + "11": 18881.0, + "12": 18329.0, + "13": 18937.0, + "14": 18987.0, + "15": 18708.0, + "16": 18957.0, + "17": 19271.0, + "18": 18855.0, + "19": 18843.0, + "20": 19091.0, + "21": 18628.0, + "22": 18905.0, + "23": 18905.0, + "24": 19000.0, + "25": 18306.0, + "26": 18509.0, + "27": 18924.0, + "28": 18259.0, + "29": 18452.0, + "30": 18270.0, + "31": 18685.0, + "32": 18898.0, + "33": 18250.0, + "34": 18394.0, + "35": 18736.0, + "36": 18582.0, + "37": 18806.0, + "38": 18396.0, + "39": 19248.0, + "40": 19332.0, + "41": 18568.0, + "42": 18144.0, + "43": 19243.0, + "44": 18642.0, + "45": 20136.0, + "46": 19403.0, + "47": 19891.0, + "48": 20076.0, + "49": 21584.0, + "50": 20152.0 } }, "mem-allocated-bytes": { @@ -118,55 +118,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388730880.0, - "2": 1388731392.0, + "1": 1388729856.0, + "2": 1388730368.0, "3": 1388729856.0, - "4": 1388727296.0, - "5": 1388729856.0, + "4": 1388727808.0, + "5": 1388728832.0, "6": 1388730368.0, - "7": 1388729856.0, + "7": 1388731904.0, "8": 1388729856.0, - "9": 1388731392.0, + "9": 1388731904.0, "10": 1388731392.0, - "11": 1388729344.0, + "11": 1388728832.0, "12": 1388728832.0, "13": 1388730880.0, "14": 1388728320.0, - "15": 1388730880.0, - "16": 1388729344.0, - "17": 1388731904.0, - "18": 1388730880.0, - "19": 1388730368.0, + "15": 1388731904.0, + "16": 1388728320.0, + "17": 1388730880.0, + "18": 1388730368.0, + "19": 1388729856.0, "20": 1388730368.0, - "21": 1388731392.0, - "22": 1388730880.0, + "21": 1388732928.0, + "22": 1388729344.0, "23": 1388731392.0, - "24": 1388730880.0, - "25": 1388731904.0, + "24": 1388729344.0, + "25": 1388732928.0, "26": 1388732928.0, - "27": 1388731392.0, - "28": 1388729856.0, + "27": 1388729344.0, + "28": 1388730368.0, "29": 1388731392.0, "30": 1388733440.0, "31": 1388732416.0, - "32": 1388730368.0, + "32": 1388729344.0, "33": 1388732416.0, "34": 1388734464.0, - "35": 1388731904.0, + "35": 1388731392.0, "36": 1388730368.0, "37": 1388731904.0, - "38": 1388729856.0, - "39": 1388732928.0, - "40": 1388733952.0, - "41": 1388729856.0, + "38": 1388730368.0, + "39": 1388733440.0, + "40": 1388734464.0, + "41": 1388730368.0, "42": 1388732928.0, "43": 1388731392.0, "44": 1388729856.0, - "45": 1388733952.0, - "46": 1388731392.0, - "47": 1388732928.0, + "45": 1388732928.0, + "46": 1388731904.0, + "47": 1388733952.0, "48": 1388731904.0, - "49": 1388736000.0, + "49": 1388736512.0, "50": 1388731904.0 } }, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179614208.0, - "2": 3661343744.0, - "3": 3663287296.0, - "4": 3663287296.0, - "5": 3663287296.0, - "6": 3663287296.0, - "7": 3663287296.0, - "8": 3663287296.0, - "9": 3663287296.0, - "10": 3663287296.0, - "11": 3663287296.0, - "12": 3663287296.0, - "13": 3663287296.0, - "14": 3663287296.0, - "15": 3663287296.0, - "16": 3663287296.0, - "17": 3663287296.0, - "18": 3663287296.0, - "19": 3663287296.0, - "20": 3663287296.0, - "21": 3663287296.0, - "22": 3663287296.0, - "23": 3663287296.0, - "24": 3663287296.0, - "25": 3663287296.0, - "26": 3663287296.0, - "27": 3663287296.0, - "28": 3663287296.0, - "29": 3663287296.0, - "30": 3663287296.0, - "31": 3663287296.0, - "32": 3663287296.0, - "33": 3663287296.0, - "34": 3664704512.0, - "35": 3664704512.0, - "36": 3664704512.0, - "37": 3664704512.0, - "38": 3664704512.0, - "39": 3664704512.0, - "40": 3664704512.0, - "41": 3664704512.0, - "42": 3664704512.0, - "43": 3664704512.0, - "44": 3664704512.0, - "45": 3664704512.0, - "46": 3665018368.0, - "47": 3665018368.0, - "48": 3665595904.0, - "49": 3665595904.0, - "50": 3665595904.0 + "1": 3179474432.0, + "2": 3662153216.0, + "3": 3662153216.0, + "4": 3662153216.0, + "5": 3662153216.0, + "6": 3662153216.0, + "7": 3662153216.0, + "8": 3662153216.0, + "9": 3662153216.0, + "10": 3662261248.0, + "11": 3662261248.0, + "12": 3662261248.0, + "13": 3662261248.0, + "14": 3662261248.0, + "15": 3662261248.0, + "16": 3662261248.0, + "17": 3662261248.0, + "18": 3662261248.0, + "19": 3662745600.0, + "20": 3662745600.0, + "21": 3662745600.0, + "22": 3662745600.0, + "23": 3662745600.0, + "24": 3662745600.0, + "25": 3662901760.0, + "26": 3663353344.0, + "27": 3663353344.0, + "28": 3663353344.0, + "29": 3663353344.0, + "30": 3663353344.0, + "31": 3663353344.0, + "32": 3663353344.0, + "33": 3663353344.0, + "34": 3663353344.0, + "35": 3663353344.0, + "36": 3663353344.0, + "37": 3663353344.0, + "38": 3663353344.0, + "39": 3664240640.0, + "40": 3664613376.0, + "41": 3664613376.0, + "42": 3664613376.0, + "43": 3664613376.0, + "44": 3664749056.0, + "45": 3664749056.0, + "46": 3664804352.0, + "47": 3664804352.0, + "48": 3664804352.0, + "49": 3664804352.0, + "50": 3664804352.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.34092, - "3": 1.58696, - "4": 1.54545, - "5": 0.82593, - "6": 1.57399, - "7": 1.36079, - "8": 1.55567, - "9": 1.67303, - "10": 1.17161, - "11": 1.37021, - "12": 1.29365, - "13": 1.13607, - "14": 1.7201, - "15": 0.98492, - "16": 1.29971, - "17": 1.38451, - "18": 1.29949, - "19": 1.23528, - "20": 1.32649, - "21": 1.12039, - "22": 1.20629, - "23": 1.29693, - "24": 2.1645, - "25": 0.71347, - "26": 1.44727, - "27": 1.23095, - "28": 1.01048, - "29": 1.37169, - "30": 1.09444, - "31": 0.92601, - "32": 1.99775, - "33": 1.07593, - "34": 0.64632, - "35": 1.48535, - "36": 1.51704, - "37": 1.27474, - "38": 0.76789, - "39": 1.54033, - "40": 1.66902, - "41": 1.19572, - "42": 1.46501, - "43": 1.15658, - "44": 1.90014, - "45": 0.77048, - "46": 1.07968, - "47": 1.28398, - "48": 1.29999, - "49": 1.10163, - "50": 1.3568 + "2": 5.20784, + "3": 1.43448, + "4": 1.21084, + "5": 0.7539, + "6": 1.25478, + "7": 1.04229, + "8": 1.45692, + "9": 1.37591, + "10": 0.94297, + "11": 1.16043, + "12": 1.28073, + "13": 1.14156, + "14": 1.51477, + "15": 0.93778, + "16": 1.19728, + "17": 1.25137, + "18": 1.27035, + "19": 1.24365, + "20": 1.27374, + "21": 1.23968, + "22": 1.17465, + "23": 1.31672, + "24": 1.83028, + "25": 0.77483, + "26": 1.07076, + "27": 1.10715, + "28": 1.34774, + "29": 1.04849, + "30": 1.0219, + "31": 0.8674, + "32": 1.90703, + "33": 1.09326, + "34": 0.57775, + "35": 1.50203, + "36": 1.48698, + "37": 1.22003, + "38": 0.76887, + "39": 1.24409, + "40": 1.7222, + "41": 1.18161, + "42": 1.35098, + "43": 1.25816, + "44": 1.46981, + "45": 0.76076, + "46": 1.1465, + "47": 1.2897, + "48": 1.36865, + "49": 1.14296, + "50": 1.71695 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json index e5622d351fb..d44ab58e15e 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.91962, - "2": 10.9168, - "3": 10.9211, - "4": 10.9202, - "5": 10.92774, - "6": 10.91636, - "7": 10.91825, - "8": 10.91825, - "9": 10.91448, - "10": 10.91644, - "11": 10.90375, - "12": 10.91532, - "13": 10.89542, - "14": 10.89837, - "15": 10.88246, - "16": 10.86562, - "17": 10.86327, - "18": 10.87059, - "19": 10.86132, - "20": 10.79564, - "21": 10.78115, - "22": 10.76223, - "23": 10.75418, - "24": 10.71745, - "25": 10.71507, - "26": 10.70251, - "27": 10.68134, - "28": 10.61348, - "29": 10.5756, - "30": 10.55196, - "31": 10.54825, - "32": 10.51901, - "33": 10.48575, - "34": 10.45833, - "35": 10.46344, - "36": 10.44142, - "37": 10.39933, - "38": 10.40604, - "39": 10.36502, - "40": 10.35425, - "41": 10.33201, - "42": 10.32538, - "43": 10.30184, - "44": 10.26353, - "45": 10.28155, - "46": 10.24406, - "47": 10.23118, - "48": 10.19372, - "49": 10.18577, - "50": 10.1917 + "1": 10.91966, + "2": 10.91726, + "3": 10.92132, + "4": 10.92035, + "5": 10.92817, + "6": 10.91641, + "7": 10.91804, + "8": 10.91811, + "9": 10.91453, + "10": 10.91661, + "11": 10.90391, + "12": 10.91493, + "13": 10.89584, + "14": 10.89893, + "15": 10.88207, + "16": 10.86474, + "17": 10.86288, + "18": 10.87074, + "19": 10.8622, + "20": 10.79575, + "21": 10.78195, + "22": 10.76221, + "23": 10.75467, + "24": 10.7176, + "25": 10.71567, + "26": 10.70228, + "27": 10.68101, + "28": 10.61305, + "29": 10.5759, + "30": 10.55078, + "31": 10.54866, + "32": 10.51944, + "33": 10.4861, + "34": 10.45799, + "35": 10.46346, + "36": 10.44157, + "37": 10.39981, + "38": 10.40598, + "39": 10.36498, + "40": 10.35539, + "41": 10.332, + "42": 10.32524, + "43": 10.30185, + "44": 10.26362, + "45": 10.28123, + "46": 10.24404, + "47": 10.23119, + "48": 10.19333, + "49": 10.18598, + "50": 10.19193 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 16504.0, - "2": 16572.0, - "3": 16848.0, - "4": 16537.0, - "5": 16489.0, - "6": 16124.0, - "7": 16810.0, - "8": 16150.0, - "9": 16548.0, - "10": 17006.0, - "11": 16733.0, - "12": 16149.0, - "13": 16785.0, - "14": 16595.0, - "15": 15935.0, - "16": 16182.0, - "17": 16600.0, - "18": 16851.0, - "19": 16582.0, - "20": 16488.0, - "21": 16536.0, - "22": 16707.0, - "23": 16806.0, - "24": 17135.0, - "25": 16344.0, - "26": 16797.0, - "27": 16755.0, - "28": 16541.0, - "29": 16348.0, - "30": 16485.0, - "31": 17033.0, - "32": 17299.0, - "33": 17333.0, - "34": 17176.0, - "35": 17567.0, - "36": 17050.0, - "37": 17719.0, - "38": 17231.0, - "39": 18134.0, - "40": 18748.0, - "41": 17574.0, - "42": 17132.0, - "43": 18791.0, - "44": 18164.0, - "45": 20261.0, - "46": 19601.0, - "47": 19299.0, - "48": 19756.0, - "49": 22082.0, - "50": 20481.0 + "1": 16524.0, + "2": 16508.0, + "3": 16785.0, + "4": 16926.0, + "5": 16462.0, + "6": 16102.0, + "7": 16748.0, + "8": 16199.0, + "9": 16613.0, + "10": 16943.0, + "11": 16425.0, + "12": 16180.0, + "13": 16657.0, + "14": 16782.0, + "15": 16148.0, + "16": 16106.0, + "17": 16492.0, + "18": 16533.0, + "19": 16530.0, + "20": 16763.0, + "21": 16445.0, + "22": 16773.0, + "23": 16630.0, + "24": 16701.0, + "25": 16064.0, + "26": 17085.0, + "27": 16732.0, + "28": 16456.0, + "29": 16464.0, + "30": 16595.0, + "31": 16897.0, + "32": 17338.0, + "33": 17253.0, + "34": 16885.0, + "35": 17615.0, + "36": 17281.0, + "37": 17687.0, + "38": 17043.0, + "39": 18394.0, + "40": 18808.0, + "41": 17518.0, + "42": 17212.0, + "43": 19063.0, + "44": 17953.0, + "45": 20054.0, + "46": 19544.0, + "47": 19211.0, + "48": 19825.0, + "49": 22059.0, + "50": 20151.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1561137664.0, - "2": 1561081856.0, - "3": 1561233920.0, - "4": 1561094144.0, - "5": 1561165312.0, - "6": 1561183232.0, - "7": 1562110464.0, - "8": 1562260480.0, - "9": 1561183232.0, - "10": 1561138176.0, - "11": 1561670144.0, - "12": 1561515520.0, - "13": 1561385984.0, - "14": 1561284608.0, - "15": 1561436672.0, - "16": 1561284608.0, - "17": 1562002944.0, - "18": 1561442304.0, - "19": 1561391616.0, - "20": 1561487360.0, - "21": 1561539584.0, - "22": 1561284608.0, - "23": 1561492992.0, - "24": 1561492992.0, - "25": 1561715712.0, - "26": 1561487360.0, - "27": 1562130944.0, - "28": 1561621504.0, - "29": 1562286592.0, - "30": 1562626048.0, - "31": 1562769920.0, - "32": 1561718784.0, - "33": 1561633792.0, - "34": 1561487360.0, - "35": 1561639424.0, - "36": 1561639424.0, - "37": 1561791488.0, - "38": 1561690112.0, - "39": 1561842176.0, - "40": 1561832448.0, - "41": 1563395584.0, - "42": 1561847808.0, - "43": 1563095552.0, - "44": 1562469376.0, - "45": 1561847808.0, - "46": 1561931776.0, - "47": 1561898496.0, - "48": 1562890752.0, - "49": 1562391552.0, - "50": 1562682368.0 + "1": 1557884416.0, + "2": 1557401600.0, + "3": 1557401600.0, + "4": 1557401600.0, + "5": 1557401600.0, + "6": 1557456896.0, + "7": 1557401600.0, + "8": 1557401600.0, + "9": 1557545984.0, + "10": 1558419456.0, + "11": 1557401600.0, + "12": 1557401600.0, + "13": 1557401600.0, + "14": 1557401600.0, + "15": 1557401600.0, + "16": 1557401600.0, + "17": 1557401600.0, + "18": 1557991424.0, + "19": 1557401600.0, + "20": 1557401600.0, + "21": 1557401600.0, + "22": 1557401600.0, + "23": 1557401600.0, + "24": 1557588992.0, + "25": 1557683200.0, + "26": 1557620736.0, + "27": 1557401600.0, + "28": 1557401600.0, + "29": 1557401600.0, + "30": 1557401600.0, + "31": 1557401600.0, + "32": 1557401600.0, + "33": 1557798912.0, + "34": 1557988352.0, + "35": 1557401600.0, + "36": 1557680128.0, + "37": 1557401600.0, + "38": 1557401600.0, + "39": 1557598208.0, + "40": 1557401600.0, + "41": 1558188032.0, + "42": 1557401600.0, + "43": 1557401600.0, + "44": 1557561344.0, + "45": 1558450176.0, + "46": 1557401600.0, + "47": 1557647360.0, + "48": 1557401600.0, + "49": 1557401600.0, + "50": 1559008256.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3488102400.0, - "2": 4062251520.0, - "3": 4062251520.0, - "4": 4062251520.0, - "5": 4062251520.0, - "6": 4068555776.0, - "7": 4068555776.0, - "8": 4073248256.0, - "9": 4073248256.0, - "10": 4073248256.0, - "11": 4073248256.0, - "12": 4073248256.0, - "13": 4073248256.0, - "14": 4073248256.0, - "15": 4073248256.0, - "16": 4073248256.0, - "17": 4073248256.0, - "18": 4073248256.0, - "19": 4073248256.0, - "20": 4073248256.0, - "21": 4073248256.0, - "22": 4073248256.0, - "23": 4073248256.0, - "24": 4073248256.0, - "25": 4073248256.0, - "26": 4073248256.0, - "27": 4073248256.0, - "28": 4073248256.0, - "29": 4073248256.0, - "30": 4073248256.0, - "31": 4073248256.0, - "32": 4073248256.0, - "33": 4073248256.0, - "34": 4073248256.0, - "35": 4073248256.0, - "36": 4073248256.0, - "37": 4073248256.0, - "38": 4073248256.0, - "39": 4073248256.0, - "40": 4073248256.0, - "41": 4073248256.0, - "42": 4073248256.0, - "43": 4073248256.0, - "44": 4073248256.0, - "45": 4073248256.0, - "46": 4073248256.0, - "47": 4073248256.0, - "48": 4073248256.0, - "49": 4073248256.0, - "50": 4073248256.0 + "1": 3492855808.0, + "2": 4056381952.0, + "3": 4056381952.0, + "4": 4056381952.0, + "5": 4056381952.0, + "6": 4068074496.0, + "7": 4068074496.0, + "8": 4073879040.0, + "9": 4073879040.0, + "10": 4073879040.0, + "11": 4073879040.0, + "12": 4073879040.0, + "13": 4073879040.0, + "14": 4073879040.0, + "15": 4073879040.0, + "16": 4073879040.0, + "17": 4073879040.0, + "18": 4073879040.0, + "19": 4073879040.0, + "20": 4073879040.0, + "21": 4073879040.0, + "22": 4073879040.0, + "23": 4073879040.0, + "24": 4073879040.0, + "25": 4073879040.0, + "26": 4073879040.0, + "27": 4073879040.0, + "28": 4073879040.0, + "29": 4073879040.0, + "30": 4073879040.0, + "31": 4073879040.0, + "32": 4073879040.0, + "33": 4073879040.0, + "34": 4073879040.0, + "35": 4073879040.0, + "36": 4073879040.0, + "37": 4073879040.0, + "38": 4073879040.0, + "39": 4073879040.0, + "40": 4073879040.0, + "41": 4073879040.0, + "42": 4073879040.0, + "43": 4073879040.0, + "44": 4073879040.0, + "45": 4073879040.0, + "46": 4073879040.0, + "47": 4073879040.0, + "48": 4073879040.0, + "49": 4073879040.0, + "50": 4073879040.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.79622, - "3": 1.21007, - "4": 1.14826, - "5": 0.83739, - "6": 0.99678, - "7": 1.06431, - "8": 1.46855, - "9": 1.45913, - "10": 0.92306, - "11": 1.2195, - "12": 1.29215, - "13": 0.85213, - "14": 1.5355, - "15": 0.84514, - "16": 1.16531, - "17": 1.04874, - "18": 1.20965, - "19": 1.05465, - "20": 1.12464, - "21": 1.14642, - "22": 1.08156, - "23": 1.09043, - "24": 1.72097, - "25": 0.8303, - "26": 1.08398, - "27": 1.03852, - "28": 0.99051, - "29": 1.08482, - "30": 0.88763, - "31": 0.84441, - "32": 1.78145, - "33": 1.01149, - "34": 0.84605, - "35": 1.10834, - "36": 1.11263, - "37": 0.94746, - "38": 0.83858, - "39": 1.1962, - "40": 1.45473, - "41": 1.0538, - "42": 1.10087, - "43": 1.00773, - "44": 1.28551, - "45": 0.94447, - "46": 1.09943, - "47": 0.95418, - "48": 1.16128, - "49": 0.98063, - "50": 1.10117 + "2": 6.69475, + "3": 1.38178, + "4": 1.14803, + "5": 0.76484, + "6": 1.0427, + "7": 1.06685, + "8": 1.56258, + "9": 1.8472, + "10": 0.70231, + "11": 1.19696, + "12": 1.4504, + "13": 0.82545, + "14": 1.81568, + "15": 0.86371, + "16": 1.19555, + "17": 1.3226, + "18": 1.32753, + "19": 1.14107, + "20": 1.0955, + "21": 1.18619, + "22": 1.08347, + "23": 1.09024, + "24": 1.65534, + "25": 0.97544, + "26": 0.78518, + "27": 1.01314, + "28": 1.05448, + "29": 1.2595, + "30": 0.88557, + "31": 0.76508, + "32": 1.87171, + "33": 1.01369, + "34": 0.72612, + "35": 1.24221, + "36": 1.11018, + "37": 1.00116, + "38": 0.70948, + "39": 1.27798, + "40": 1.46193, + "41": 1.03174, + "42": 1.17941, + "43": 1.0079, + "44": 1.31564, + "45": 0.73439, + "46": 1.14061, + "47": 1.13838, + "48": 1.02369, + "49": 0.94855, + "50": 1.23548 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json index 38dc0e7bfba..2e11504a7b5 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94001, - "2": 10.92278, - "3": 10.93015, - "4": 10.92656, - "5": 10.93629, - "6": 10.9223, - "7": 10.92716, - "8": 10.92802, - "9": 10.93426, - "10": 10.92691, - "11": 10.92376, - "12": 10.92415, - "13": 10.91326, - "14": 10.91511, - "15": 10.89886, - "16": 10.88689, - "17": 10.8878, - "18": 10.88188, - "19": 10.88572, - "20": 10.8085, - "21": 10.8079, - "22": 10.78818, - "23": 10.78763, - "24": 10.75531, - "25": 10.75377, - "26": 10.74534, - "27": 10.72158, - "28": 10.64806, - "29": 10.62373, - "30": 10.59438, - "31": 10.59725, - "32": 10.57989, - "33": 10.544, - "34": 10.50995, - "35": 10.51437, - "36": 10.49229, - "37": 10.46454, - "38": 10.45962, - "39": 10.43573, - "40": 10.41326, - "41": 10.39006, - "42": 10.37181, - "43": 10.35634, - "44": 10.33171, - "45": 10.33854, - "46": 10.29357, - "47": 10.27812, - "48": 10.2433, - "49": 10.23396, - "50": 10.23414 + "1": 10.9393, + "2": 10.92254, + "3": 10.93003, + "4": 10.92653, + "5": 10.93663, + "6": 10.92231, + "7": 10.92692, + "8": 10.9276, + "9": 10.93447, + "10": 10.92803, + "11": 10.92337, + "12": 10.92395, + "13": 10.9138, + "14": 10.91501, + "15": 10.89836, + "16": 10.88716, + "17": 10.88789, + "18": 10.8814, + "19": 10.88578, + "20": 10.80839, + "21": 10.80736, + "22": 10.78758, + "23": 10.78703, + "24": 10.75415, + "25": 10.7537, + "26": 10.74516, + "27": 10.72174, + "28": 10.64768, + "29": 10.62346, + "30": 10.59487, + "31": 10.59731, + "32": 10.58021, + "33": 10.54459, + "34": 10.51017, + "35": 10.5141, + "36": 10.492, + "37": 10.46445, + "38": 10.45973, + "39": 10.43577, + "40": 10.41313, + "41": 10.39014, + "42": 10.3718, + "43": 10.35627, + "44": 10.33169, + "45": 10.33879, + "46": 10.29367, + "47": 10.27803, + "48": 10.24357, + "49": 10.23406, + "50": 10.23439 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5750.0, - "2": 6129.0, - "3": 5823.0, - "4": 5794.0, - "5": 5914.0, - "6": 5580.0, - "7": 6134.0, - "8": 5917.0, - "9": 5761.0, - "10": 5958.0, - "11": 5805.0, - "12": 5800.0, - "13": 6023.0, - "14": 6095.0, - "15": 5652.0, - "16": 5826.0, - "17": 5902.0, - "18": 5891.0, - "19": 5785.0, - "20": 5685.0, - "21": 5918.0, - "22": 5778.0, - "23": 5868.0, - "24": 5858.0, - "25": 5705.0, - "26": 5769.0, - "27": 5977.0, - "28": 5849.0, - "29": 5911.0, - "30": 5951.0, - "31": 6220.0, - "32": 5937.0, - "33": 5939.0, - "34": 6185.0, - "35": 6164.0, - "36": 6043.0, - "37": 6462.0, - "38": 6229.0, - "39": 6403.0, - "40": 6429.0, - "41": 6671.0, - "42": 6178.0, - "43": 6626.0, - "44": 6265.0, - "45": 7346.0, - "46": 6616.0, - "47": 6836.0, - "48": 7145.0, - "49": 7364.0, - "50": 7278.0 + "1": 5731.0, + "2": 5862.0, + "3": 5904.0, + "4": 5737.0, + "5": 5857.0, + "6": 5686.0, + "7": 5957.0, + "8": 5872.0, + "9": 5831.0, + "10": 5865.0, + "11": 5744.0, + "12": 5673.0, + "13": 5857.0, + "14": 6019.0, + "15": 5625.0, + "16": 5706.0, + "17": 5813.0, + "18": 5764.0, + "19": 5988.0, + "20": 5729.0, + "21": 5754.0, + "22": 5818.0, + "23": 5822.0, + "24": 5984.0, + "25": 5840.0, + "26": 5899.0, + "27": 5941.0, + "28": 5794.0, + "29": 6038.0, + "30": 5982.0, + "31": 6115.0, + "32": 6111.0, + "33": 5966.0, + "34": 6111.0, + "35": 6273.0, + "36": 6002.0, + "37": 6388.0, + "38": 6311.0, + "39": 6345.0, + "40": 6692.0, + "41": 6659.0, + "42": 6076.0, + "43": 6646.0, + "44": 6220.0, + "45": 7071.0, + "46": 6785.0, + "47": 6746.0, + "48": 7209.0, + "49": 7332.0, + "50": 7146.0 } }, "mem-allocated-bytes": { @@ -120,54 +120,54 @@ "values": { "1": 598356992.0, "2": 598359552.0, - "3": 598360064.0, + "3": 598359040.0, "4": 598358016.0, - "5": 598359552.0, + "5": 598359040.0, "6": 598359040.0, - "7": 598355968.0, + "7": 598356992.0, "8": 598358016.0, - "9": 598357504.0, + "9": 598356480.0, "10": 598359040.0, "11": 598358016.0, - "12": 598356992.0, + "12": 598359040.0, "13": 598359040.0, "14": 598355968.0, "15": 598359552.0, "16": 598358016.0, "17": 598359040.0, - "18": 598359040.0, + "18": 598358016.0, "19": 598358528.0, "20": 598358016.0, - "21": 598355968.0, - "22": 598358016.0, - "23": 598358528.0, - "24": 598359040.0, - "25": 598358528.0, + "21": 598356992.0, + "22": 598359040.0, + "23": 598357504.0, + "24": 598358016.0, + "25": 598359552.0, "26": 598358016.0, - "27": 598360064.0, + "27": 598361088.0, "28": 598358016.0, - "29": 598358016.0, + "29": 598359040.0, "30": 598357504.0, "31": 598358016.0, "32": 598358016.0, "33": 598359040.0, "34": 598359040.0, - "35": 598356992.0, + "35": 598358016.0, "36": 598358016.0, "37": 598358528.0, "38": 598358016.0, - "39": 598356992.0, + "39": 598358016.0, "40": 598356992.0, "41": 598355968.0, "42": 598355456.0, "43": 598356480.0, "44": 598355968.0, - "45": 598356992.0, + "45": 598358528.0, "46": 598356992.0, "47": 598356480.0, "48": 598356992.0, "49": 598357504.0, - "50": 598359552.0 + "50": 598358528.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 854438400.0, - "2": 1081444864.0, - "3": 1081444864.0, - "4": 1084059648.0, - "5": 1084059648.0, - "6": 1084059648.0, - "7": 1084059648.0, - "8": 1084059648.0, - "9": 1084059648.0, - "10": 1084059648.0, - "11": 1084059648.0, - "12": 1084059648.0, - "13": 1084059648.0, - "14": 1084059648.0, - "15": 1084059648.0, - "16": 1084059648.0, - "17": 1084489728.0, - "18": 1084489728.0, - "19": 1084489728.0, - "20": 1084489728.0, - "21": 1084489728.0, - "22": 1084489728.0, - "23": 1084489728.0, - "24": 1084489728.0, - "25": 1084489728.0, - "26": 1084489728.0, - "27": 1084489728.0, - "28": 1084489728.0, - "29": 1084489728.0, - "30": 1084489728.0, - "31": 1084489728.0, - "32": 1084489728.0, - "33": 1084489728.0, - "34": 1084489728.0, - "35": 1084489728.0, - "36": 1084489728.0, - "37": 1084489728.0, - "38": 1084489728.0, - "39": 1084489728.0, - "40": 1084489728.0, - "41": 1084489728.0, - "42": 1084489728.0, - "43": 1084489728.0, - "44": 1084489728.0, - "45": 1084489728.0, - "46": 1084489728.0, - "47": 1084489728.0, - "48": 1084489728.0, - "49": 1084489728.0, - "50": 1084489728.0 + "1": 854393344.0, + "2": 1081646080.0, + "3": 1081646080.0, + "4": 1083935744.0, + "5": 1083935744.0, + "6": 1083935744.0, + "7": 1083935744.0, + "8": 1083935744.0, + "9": 1083935744.0, + "10": 1083935744.0, + "11": 1083935744.0, + "12": 1083935744.0, + "13": 1083935744.0, + "14": 1083935744.0, + "15": 1083935744.0, + "16": 1083935744.0, + "17": 1083935744.0, + "18": 1083935744.0, + "19": 1083935744.0, + "20": 1083935744.0, + "21": 1083935744.0, + "22": 1083935744.0, + "23": 1083935744.0, + "24": 1083935744.0, + "25": 1083935744.0, + "26": 1083935744.0, + "27": 1083935744.0, + "28": 1083935744.0, + "29": 1083935744.0, + "30": 1083935744.0, + "31": 1083935744.0, + "32": 1083935744.0, + "33": 1083935744.0, + "34": 1083935744.0, + "35": 1083935744.0, + "36": 1083935744.0, + "37": 1083935744.0, + "38": 1083935744.0, + "39": 1083935744.0, + "40": 1083935744.0, + "41": 1083935744.0, + "42": 1083935744.0, + "43": 1083935744.0, + "44": 1083935744.0, + "45": 1083935744.0, + "46": 1083935744.0, + "47": 1083935744.0, + "48": 1083935744.0, + "49": 1083935744.0, + "50": 1083935744.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.65971, - "3": 0.66444, - "4": 0.64584, - "5": 0.64868, - "6": 0.64853, - "7": 0.64476, - "8": 0.64459, - "9": 0.64509, - "10": 0.64486, - "11": 0.64809, - "12": 0.65008, - "13": 0.64553, - "14": 0.64521, - "15": 0.64465, - "16": 0.6552, - "17": 0.67223, - "18": 0.65575, - "19": 0.6548, - "20": 0.65375, - "21": 0.65446, - "22": 0.65402, - "23": 0.65445, - "24": 0.65667, - "25": 0.65639, - "26": 0.65476, - "27": 0.65499, - "28": 0.65377, - "29": 0.65416, - "30": 1.13757, - "31": 0.65466, - "32": 0.65482, - "33": 0.65391, - "34": 0.65405, - "35": 0.65303, - "36": 0.65259, - "37": 0.65254, - "38": 0.65236, - "39": 0.65269, - "40": 0.65384, - "41": 0.6553, - "42": 0.65301, - "43": 0.65204, - "44": 1.15578, - "45": 0.6541, - "46": 0.65216, - "47": 0.65239, - "48": 0.65095, - "49": 0.65181, - "50": 0.65609 + "2": 7.62468, + "3": 0.69955, + "4": 0.67832, + "5": 0.68164, + "6": 0.68435, + "7": 0.68282, + "8": 0.68146, + "9": 0.6813, + "10": 0.68267, + "11": 0.68288, + "12": 0.68242, + "13": 0.68425, + "14": 0.68672, + "15": 0.68762, + "16": 0.68476, + "17": 0.68624, + "18": 0.68597, + "19": 0.68462, + "20": 0.68623, + "21": 0.68596, + "22": 0.68619, + "23": 0.6855, + "24": 0.68944, + "25": 0.68489, + "26": 0.692, + "27": 0.68595, + "28": 0.68384, + "29": 0.69701, + "30": 0.70408, + "31": 0.68352, + "32": 0.68796, + "33": 0.68548, + "34": 0.68143, + "35": 0.68555, + "36": 0.69015, + "37": 0.68303, + "38": 0.68481, + "39": 0.67882, + "40": 0.66741, + "41": 0.66614, + "42": 0.66726, + "43": 0.66745, + "44": 0.66769, + "45": 0.66616, + "46": 0.66718, + "47": 0.66685, + "48": 0.66761, + "49": 0.66623, + "50": 0.6665 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index c8d3909a023..3a2660c5183 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89844, - "2": 10.90039, - "3": 10.89837, - "4": 10.89688, - "5": 10.89885, - "6": 10.88618, - "7": 10.88978, - "8": 10.89165, - "9": 10.89445, - "10": 10.89776, - "11": 10.887, - "12": 10.89063, - "13": 10.88395, - "14": 10.87663, - "15": 10.86762, - "16": 10.85456, - "17": 10.85541, - "18": 10.83795, - "19": 10.85342, - "20": 10.78888, - "21": 10.77633, - "22": 10.77318, - "23": 10.76231, - "24": 10.73158, - "25": 10.72373, - "26": 10.72212, - "27": 10.701, - "28": 10.64017, - "29": 10.60986, - "30": 10.59138, - "31": 10.57684, - "32": 10.55902, - "33": 10.52555, - "34": 10.5102, - "35": 10.49812, - "36": 10.48247, - "37": 10.4528, - "38": 10.46112, - "39": 10.42931, - "40": 10.41237, - "41": 10.39529, - "42": 10.36917, - "43": 10.34961, - "44": 10.32146, - "45": 10.33362, - "46": 10.29354, - "47": 10.28045, - "48": 10.2359, - "49": 10.24617, - "50": 10.23514 + "1": 10.89802, + "2": 10.90021, + "3": 10.89823, + "4": 10.89669, + "5": 10.89951, + "6": 10.88562, + "7": 10.89028, + "8": 10.89225, + "9": 10.89399, + "10": 10.89786, + "11": 10.88695, + "12": 10.88994, + "13": 10.88377, + "14": 10.87719, + "15": 10.8672, + "16": 10.8546, + "17": 10.8549, + "18": 10.83796, + "19": 10.85411, + "20": 10.78881, + "21": 10.7772, + "22": 10.77399, + "23": 10.76223, + "24": 10.73066, + "25": 10.72292, + "26": 10.72271, + "27": 10.70096, + "28": 10.64012, + "29": 10.61024, + "30": 10.59081, + "31": 10.57683, + "32": 10.55893, + "33": 10.52562, + "34": 10.51045, + "35": 10.49804, + "36": 10.48157, + "37": 10.45275, + "38": 10.46063, + "39": 10.42883, + "40": 10.412, + "41": 10.39572, + "42": 10.3692, + "43": 10.34979, + "44": 10.32155, + "45": 10.33299, + "46": 10.29376, + "47": 10.2807, + "48": 10.23603, + "49": 10.24619, + "50": 10.23463 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5981.0, - "2": 5941.0, - "3": 6032.0, - "4": 6141.0, - "5": 5994.0, - "6": 5669.0, - "7": 6099.0, - "8": 5999.0, - "9": 6071.0, - "10": 6120.0, - "11": 5905.0, - "12": 5635.0, - "13": 6149.0, - "14": 6139.0, - "15": 5944.0, - "16": 5941.0, - "17": 6192.0, - "18": 6036.0, - "19": 6011.0, - "20": 5807.0, - "21": 5807.0, - "22": 5888.0, - "23": 5911.0, - "24": 6044.0, - "25": 5619.0, - "26": 5999.0, - "27": 6001.0, - "28": 5983.0, - "29": 6083.0, - "30": 6020.0, - "31": 6002.0, - "32": 6149.0, - "33": 5987.0, - "34": 6100.0, - "35": 6255.0, - "36": 6013.0, - "37": 6476.0, - "38": 6115.0, - "39": 6474.0, - "40": 6498.0, - "41": 6621.0, - "42": 6077.0, - "43": 6920.0, - "44": 6484.0, - "45": 7274.0, - "46": 6778.0, - "47": 7035.0, - "48": 7388.0, - "49": 7581.0, - "50": 7291.0 + "1": 5949.0, + "2": 5859.0, + "3": 6058.0, + "4": 6009.0, + "5": 6077.0, + "6": 5629.0, + "7": 6046.0, + "8": 6056.0, + "9": 6076.0, + "10": 6057.0, + "11": 6040.0, + "12": 5768.0, + "13": 6176.0, + "14": 6151.0, + "15": 5912.0, + "16": 5989.0, + "17": 6088.0, + "18": 5925.0, + "19": 6083.0, + "20": 5820.0, + "21": 5694.0, + "22": 5956.0, + "23": 5936.0, + "24": 6145.0, + "25": 5845.0, + "26": 5951.0, + "27": 5898.0, + "28": 6086.0, + "29": 5955.0, + "30": 5874.0, + "31": 6090.0, + "32": 5974.0, + "33": 5926.0, + "34": 6205.0, + "35": 6237.0, + "36": 6061.0, + "37": 6564.0, + "38": 6296.0, + "39": 6436.0, + "40": 6544.0, + "41": 6819.0, + "42": 6065.0, + "43": 6763.0, + "44": 6546.0, + "45": 7297.0, + "46": 6821.0, + "47": 6931.0, + "48": 7259.0, + "49": 7526.0, + "50": 7287.0 } }, "mem-allocated-bytes": { @@ -121,49 +121,49 @@ "1": 627719168.0, "2": 627717120.0, "3": 627718144.0, - "4": 627718656.0, + "4": 627719168.0, "5": 627717632.0, - "6": 627718656.0, - "7": 627718144.0, - "8": 627718144.0, - "9": 627719168.0, + "6": 627717632.0, + "7": 627719168.0, + "8": 627717120.0, + "9": 627718144.0, "10": 627717120.0, "11": 627719680.0, "12": 627718144.0, - "13": 627717632.0, + "13": 627717120.0, "14": 627720192.0, "15": 627718144.0, - "16": 627720704.0, + "16": 627719680.0, "17": 627720192.0, - "18": 627718144.0, + "18": 627719168.0, "19": 627718656.0, - "20": 627718656.0, - "21": 627717120.0, + "20": 627718144.0, + "21": 627718144.0, "22": 627718144.0, - "23": 627720192.0, + "23": 627719168.0, "24": 627716608.0, "25": 627718144.0, - "26": 627720704.0, + "26": 627720192.0, "27": 627719168.0, "28": 627719168.0, - "29": 627719680.0, + "29": 627719168.0, "30": 627719168.0, "31": 627718144.0, "32": 627720192.0, "33": 627719168.0, "34": 627719168.0, - "35": 627717632.0, - "36": 627719168.0, + "35": 627718144.0, + "36": 627718144.0, "37": 627719168.0, "38": 627719680.0, "39": 627717632.0, - "40": 627719168.0, - "41": 627719168.0, + "40": 627718144.0, + "41": 627720192.0, "42": 627718144.0, - "43": 627719680.0, - "44": 627719680.0, - "45": 627719680.0, - "46": 627720192.0, + "43": 627718656.0, + "44": 627719168.0, + "45": 627718656.0, + "46": 627720704.0, "47": 627720704.0, "48": 627718144.0, "49": 627721216.0, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 878641664.0, - "2": 1112412160.0, - "3": 1112412160.0, - "4": 1113359360.0, - "5": 1113843712.0, - "6": 1113843712.0, - "7": 1113843712.0, - "8": 1113843712.0, - "9": 1113843712.0, - "10": 1113843712.0, - "11": 1113843712.0, - "12": 1113843712.0, - "13": 1113843712.0, - "14": 1113843712.0, - "15": 1113843712.0, - "16": 1113843712.0, - "17": 1113843712.0, - "18": 1113843712.0, - "19": 1113843712.0, - "20": 1113843712.0, - "21": 1113843712.0, - "22": 1113843712.0, - "23": 1113843712.0, - "24": 1113843712.0, - "25": 1113843712.0, - "26": 1113843712.0, - "27": 1113843712.0, - "28": 1113843712.0, - "29": 1113843712.0, - "30": 1113843712.0, - "31": 1113843712.0, - "32": 1113843712.0, - "33": 1113843712.0, - "34": 1113843712.0, - "35": 1113843712.0, - "36": 1113843712.0, - "37": 1113843712.0, - "38": 1113843712.0, - "39": 1113843712.0, - "40": 1113843712.0, - "41": 1113843712.0, - "42": 1113843712.0, - "43": 1113843712.0, - "44": 1113843712.0, - "45": 1113843712.0, - "46": 1113843712.0, - "47": 1113843712.0, - "48": 1113843712.0, - "49": 1113843712.0, - "50": 1113843712.0 + "1": 880758784.0, + "2": 1113250816.0, + "3": 1113250816.0, + "4": 1114362880.0, + "5": 1114579968.0, + "6": 1114579968.0, + "7": 1114579968.0, + "8": 1114579968.0, + "9": 1114579968.0, + "10": 1114579968.0, + "11": 1114579968.0, + "12": 1114579968.0, + "13": 1114579968.0, + "14": 1114579968.0, + "15": 1114579968.0, + "16": 1114579968.0, + "17": 1114579968.0, + "18": 1114579968.0, + "19": 1114579968.0, + "20": 1114579968.0, + "21": 1114579968.0, + "22": 1114579968.0, + "23": 1114579968.0, + "24": 1114579968.0, + "25": 1114579968.0, + "26": 1114579968.0, + "27": 1114579968.0, + "28": 1114579968.0, + "29": 1114579968.0, + "30": 1114579968.0, + "31": 1114579968.0, + "32": 1114579968.0, + "33": 1114579968.0, + "34": 1114937344.0, + "35": 1114937344.0, + "36": 1114937344.0, + "37": 1114937344.0, + "38": 1114937344.0, + "39": 1114937344.0, + "40": 1114937344.0, + "41": 1114937344.0, + "42": 1114937344.0, + "43": 1114937344.0, + "44": 1114937344.0, + "45": 1114937344.0, + "46": 1114937344.0, + "47": 1114937344.0, + "48": 1114937344.0, + "49": 1114937344.0, + "50": 1114937344.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 10.09831, - "3": 0.83288, - "4": 0.80202, - "5": 0.81271, - "6": 0.80333, - "7": 0.80483, - "8": 0.79909, - "9": 0.79933, - "10": 0.8504, - "11": 0.89286, - "12": 0.79748, - "13": 0.79967, - "14": 0.80239, - "15": 0.80131, - "16": 0.79602, - "17": 0.80048, - "18": 0.79555, - "19": 0.79477, - "20": 0.79815, - "21": 0.80422, - "22": 0.80129, - "23": 0.8009, - "24": 0.80057, - "25": 0.79897, - "26": 0.78905, - "27": 0.79903, - "28": 0.79433, - "29": 0.80022, - "30": 0.79744, - "31": 0.79997, - "32": 0.79956, - "33": 0.79632, - "34": 0.79575, - "35": 0.79526, - "36": 0.79789, - "37": 0.79737, - "38": 0.79741, - "39": 0.79638, - "40": 0.79141, - "41": 0.79849, - "42": 0.79352, - "43": 0.81171, - "44": 0.79683, - "45": 0.79275, - "46": 0.78948, - "47": 0.79408, - "48": 0.79284, - "49": 0.79534, - "50": 0.80018 + "2": 15.8589, + "3": 0.77349, + "4": 0.69172, + "5": 0.68829, + "6": 0.69622, + "7": 0.68923, + "8": 0.68678, + "9": 0.68379, + "10": 0.67651, + "11": 0.68125, + "12": 0.68286, + "13": 0.6809, + "14": 0.68836, + "15": 0.68465, + "16": 0.69305, + "17": 0.68448, + "18": 0.68959, + "19": 0.68789, + "20": 0.68605, + "21": 0.68866, + "22": 0.68617, + "23": 0.68476, + "24": 0.67306, + "25": 0.68044, + "26": 0.68316, + "27": 0.68308, + "28": 0.68193, + "29": 0.67802, + "30": 0.68169, + "31": 0.68192, + "32": 0.67936, + "33": 0.67698, + "34": 0.67554, + "35": 0.66994, + "36": 0.67465, + "37": 0.66518, + "38": 0.67493, + "39": 0.67486, + "40": 0.68845, + "41": 0.67628, + "42": 0.68122, + "43": 0.68529, + "44": 0.67454, + "45": 0.67626, + "46": 0.67827, + "47": 0.67314, + "48": 0.68273, + "49": 0.68276, + "50": 0.67534 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json index aa3ae9e005a..bc66439f7e7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93494, - "2": 10.92216, - "3": 10.9323, - "4": 10.93809, - "5": 10.93865, - "6": 10.9346, - "7": 10.93441, - "8": 10.93163, - "9": 10.93122, - "10": 10.93518, - "11": 10.92054, - "12": 10.92608, - "13": 10.90642, - "14": 10.90507, - "15": 10.909, - "16": 10.88742, - "17": 10.88342, - "18": 10.88483, - "19": 10.87524, - "20": 10.81739, - "21": 10.8076, - "22": 10.79095, - "23": 10.7958, - "24": 10.76742, - "25": 10.76749, - "26": 10.74956, - "27": 10.73015, - "28": 10.6667, - "29": 10.63232, - "30": 10.60781, - "31": 10.61094, - "32": 10.5864, - "33": 10.56805, - "34": 10.52557, - "35": 10.53216, - "36": 10.50614, - "37": 10.48124, - "38": 10.48137, - "39": 10.45046, - "40": 10.42825, - "41": 10.41243, - "42": 10.39717, - "43": 10.3757, - "44": 10.3515, - "45": 10.35266, - "46": 10.32123, - "47": 10.3062, - "48": 10.26422, - "49": 10.25789, - "50": 10.25794 + "1": 10.93497, + "2": 10.92175, + "3": 10.93235, + "4": 10.9375, + "5": 10.93907, + "6": 10.93498, + "7": 10.93453, + "8": 10.93082, + "9": 10.93185, + "10": 10.93498, + "11": 10.92081, + "12": 10.92683, + "13": 10.90659, + "14": 10.90486, + "15": 10.9094, + "16": 10.88686, + "17": 10.88327, + "18": 10.88527, + "19": 10.87588, + "20": 10.81635, + "21": 10.80735, + "22": 10.79173, + "23": 10.79525, + "24": 10.76741, + "25": 10.76686, + "26": 10.74909, + "27": 10.73074, + "28": 10.66624, + "29": 10.63199, + "30": 10.60767, + "31": 10.61074, + "32": 10.58623, + "33": 10.56862, + "34": 10.52586, + "35": 10.53161, + "36": 10.5062, + "37": 10.48115, + "38": 10.4817, + "39": 10.45107, + "40": 10.42809, + "41": 10.41226, + "42": 10.39693, + "43": 10.37553, + "44": 10.35212, + "45": 10.35281, + "46": 10.32101, + "47": 10.30626, + "48": 10.26415, + "49": 10.25805, + "50": 10.25789 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5987.0, - "2": 5977.0, - "3": 6039.0, - "4": 5938.0, - "5": 5893.0, - "6": 5729.0, - "7": 6093.0, - "8": 6012.0, - "9": 6061.0, - "10": 6204.0, - "11": 5930.0, - "12": 5870.0, - "13": 6107.0, - "14": 6074.0, - "15": 5913.0, - "16": 5867.0, - "17": 6269.0, - "18": 5830.0, - "19": 5849.0, - "20": 5813.0, - "21": 6148.0, - "22": 5882.0, - "23": 5984.0, - "24": 5934.0, - "25": 5886.0, - "26": 5894.0, - "27": 6105.0, - "28": 6007.0, - "29": 6168.0, - "30": 5987.0, - "31": 6024.0, - "32": 6208.0, - "33": 5977.0, - "34": 6163.0, - "35": 6319.0, - "36": 6098.0, - "37": 6401.0, - "38": 6289.0, - "39": 6429.0, - "40": 6665.0, - "41": 6804.0, - "42": 6160.0, - "43": 6544.0, - "44": 6378.0, - "45": 6863.0, - "46": 6693.0, - "47": 6960.0, - "48": 6986.0, - "49": 7337.0, - "50": 7059.0 + "1": 5856.0, + "2": 5991.0, + "3": 6013.0, + "4": 6135.0, + "5": 5999.0, + "6": 5719.0, + "7": 6256.0, + "8": 6017.0, + "9": 6094.0, + "10": 5941.0, + "11": 6021.0, + "12": 5790.0, + "13": 6108.0, + "14": 6116.0, + "15": 5881.0, + "16": 6019.0, + "17": 6028.0, + "18": 5941.0, + "19": 6094.0, + "20": 5835.0, + "21": 5923.0, + "22": 5942.0, + "23": 5903.0, + "24": 5962.0, + "25": 5788.0, + "26": 6009.0, + "27": 6060.0, + "28": 6069.0, + "29": 6211.0, + "30": 6017.0, + "31": 5972.0, + "32": 6087.0, + "33": 5956.0, + "34": 6038.0, + "35": 6424.0, + "36": 6113.0, + "37": 6390.0, + "38": 6259.0, + "39": 6430.0, + "40": 6691.0, + "41": 6741.0, + "42": 6045.0, + "43": 6576.0, + "44": 6446.0, + "45": 6940.0, + "46": 6925.0, + "47": 6897.0, + "48": 7075.0, + "49": 7472.0, + "50": 7057.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 627719168.0, - "2": 627717120.0, - "3": 627719168.0, - "4": 627717120.0, - "5": 627718144.0, - "6": 627719168.0, - "7": 627719168.0, - "8": 627717632.0, - "9": 627718656.0, - "10": 627719168.0, - "11": 627718656.0, - "12": 627717120.0, - "13": 627718144.0, - "14": 627719168.0, - "15": 627716096.0, - "16": 627717632.0, - "17": 627718144.0, - "18": 627719680.0, - "19": 627718144.0, - "20": 627718144.0, - "21": 627717632.0, - "22": 627718144.0, - "23": 627718656.0, - "24": 627718144.0, - "25": 627719168.0, - "26": 627717632.0, - "27": 627718656.0, - "28": 627719680.0, - "29": 627718144.0, - "30": 627718656.0, - "31": 627718656.0, - "32": 627718144.0, - "33": 627716096.0, - "34": 627719680.0, - "35": 627718144.0, - "36": 627719680.0, - "37": 627716608.0, - "38": 627717120.0, - "39": 627717632.0, - "40": 627718144.0, - "41": 627717120.0, - "42": 627717632.0, - "43": 627717120.0, - "44": 627717120.0, - "45": 627716608.0, - "46": 627718144.0, - "47": 627717120.0, - "48": 627718144.0, - "49": 627718144.0, - "50": 627717120.0 + "1": 628506624.0, + "2": 628504576.0, + "3": 628505600.0, + "4": 628503552.0, + "5": 628504576.0, + "6": 628504576.0, + "7": 628505600.0, + "8": 628504064.0, + "9": 628505088.0, + "10": 628505600.0, + "11": 628504576.0, + "12": 628503552.0, + "13": 628504064.0, + "14": 628504576.0, + "15": 628504576.0, + "16": 628503552.0, + "17": 628504064.0, + "18": 628506112.0, + "19": 628504576.0, + "20": 628504576.0, + "21": 628505600.0, + "22": 628504576.0, + "23": 628504576.0, + "24": 628505600.0, + "25": 628505600.0, + "26": 628505088.0, + "27": 628504064.0, + "28": 628506112.0, + "29": 628504576.0, + "30": 628504576.0, + "31": 628504576.0, + "32": 628505600.0, + "33": 628501504.0, + "34": 628506112.0, + "35": 628504576.0, + "36": 628504576.0, + "37": 628503040.0, + "38": 628503552.0, + "39": 628503552.0, + "40": 628505088.0, + "41": 628503552.0, + "42": 628503040.0, + "43": 628503552.0, + "44": 628503552.0, + "45": 628503552.0, + "46": 628503552.0, + "47": 628503552.0, + "48": 628505600.0, + "49": 628504576.0, + "50": 628503552.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 880525312.0, - "2": 1112519168.0, - "3": 1112519168.0, - "4": 1112519168.0, - "5": 1112898048.0, - "6": 1112898048.0, - "7": 1112898048.0, - "8": 1113367552.0, - "9": 1113367552.0, - "10": 1113367552.0, - "11": 1113367552.0, - "12": 1113367552.0, - "13": 1113367552.0, - "14": 1113367552.0, - "15": 1113367552.0, - "16": 1113367552.0, - "17": 1113367552.0, - "18": 1113367552.0, - "19": 1113620992.0, - "20": 1114435584.0, - "21": 1114435584.0, - "22": 1114435584.0, - "23": 1114435584.0, - "24": 1114435584.0, - "25": 1114435584.0, - "26": 1114435584.0, - "27": 1114435584.0, - "28": 1114435584.0, - "29": 1114435584.0, - "30": 1114435584.0, - "31": 1114435584.0, - "32": 1114435584.0, - "33": 1114435584.0, - "34": 1114435584.0, - "35": 1114435584.0, - "36": 1114435584.0, - "37": 1114435584.0, - "38": 1114435584.0, - "39": 1114435584.0, - "40": 1114435584.0, - "41": 1114435584.0, - "42": 1114435584.0, - "43": 1114435584.0, - "44": 1114435584.0, - "45": 1114435584.0, - "46": 1114435584.0, - "47": 1114435584.0, - "48": 1114435584.0, - "49": 1114435584.0, - "50": 1114435584.0 + "1": 879522304.0, + "2": 1113401856.0, + "3": 1113401856.0, + "4": 1113401856.0, + "5": 1113401856.0, + "6": 1113401856.0, + "7": 1113401856.0, + "8": 1113740288.0, + "9": 1113740288.0, + "10": 1113774592.0, + "11": 1113774592.0, + "12": 1113774592.0, + "13": 1113774592.0, + "14": 1113774592.0, + "15": 1113774592.0, + "16": 1113774592.0, + "17": 1113774592.0, + "18": 1113774592.0, + "19": 1114180096.0, + "20": 1114966528.0, + "21": 1114966528.0, + "22": 1114966528.0, + "23": 1114966528.0, + "24": 1114966528.0, + "25": 1114966528.0, + "26": 1114966528.0, + "27": 1114966528.0, + "28": 1114966528.0, + "29": 1114966528.0, + "30": 1114966528.0, + "31": 1114966528.0, + "32": 1114966528.0, + "33": 1114966528.0, + "34": 1114966528.0, + "35": 1114966528.0, + "36": 1114966528.0, + "37": 1114966528.0, + "38": 1114966528.0, + "39": 1114966528.0, + "40": 1114966528.0, + "41": 1114966528.0, + "42": 1114966528.0, + "43": 1114966528.0, + "44": 1114966528.0, + "45": 1114966528.0, + "46": 1114966528.0, + "47": 1114966528.0, + "48": 1114966528.0, + "49": 1114966528.0, + "50": 1114966528.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.76246, - "3": 0.50034, - "4": 0.4652, - "5": 0.45522, - "6": 0.45718, - "7": 0.45189, - "8": 0.45941, - "9": 0.44979, - "10": 0.45091, - "11": 0.45309, - "12": 0.45487, - "13": 0.45147, - "14": 0.44874, - "15": 0.45259, - "16": 0.44792, - "17": 0.44871, - "18": 0.47429, - "19": 0.45709, - "20": 0.44938, - "21": 0.45699, - "22": 0.46948, - "23": 0.4519, - "24": 0.44958, - "25": 0.44938, - "26": 0.46043, - "27": 0.44929, - "28": 0.44955, - "29": 0.44838, - "30": 0.44659, - "31": 0.44941, - "32": 0.44391, - "33": 0.44683, - "34": 0.44531, - "35": 0.45884, - "36": 0.44718, - "37": 0.44736, - "38": 0.44632, - "39": 0.45086, - "40": 0.4464, - "41": 0.45008, - "42": 0.4472, - "43": 0.45228, - "44": 0.45277, - "45": 0.44771, - "46": 0.44694, - "47": 0.44779, - "48": 0.44774, - "49": 0.45323, - "50": 0.44931 + "2": 10.78268, + "3": 0.52849, + "4": 0.50108, + "5": 0.49375, + "6": 0.49549, + "7": 0.48606, + "8": 0.48801, + "9": 0.48313, + "10": 0.48733, + "11": 0.48391, + "12": 0.4956, + "13": 0.48357, + "14": 0.4887, + "15": 0.48398, + "16": 0.48424, + "17": 0.48821, + "18": 0.49376, + "19": 0.48713, + "20": 0.48491, + "21": 0.48579, + "22": 0.48864, + "23": 0.48815, + "24": 0.48739, + "25": 0.48481, + "26": 0.49352, + "27": 0.48464, + "28": 0.4839, + "29": 0.48585, + "30": 0.48541, + "31": 0.48581, + "32": 0.48627, + "33": 0.48596, + "34": 0.48485, + "35": 0.48687, + "36": 0.48471, + "37": 0.48481, + "38": 0.4833, + "39": 0.49697, + "40": 0.48121, + "41": 0.48778, + "42": 0.49158, + "43": 0.48666, + "44": 0.48513, + "45": 0.48381, + "46": 0.48232, + "47": 0.48891, + "48": 0.48126, + "49": 0.49084, + "50": 0.47878 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json index 68cf5ade642..6e731ccca6c 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94001, - "2": 10.92278, - "3": 10.93015, - "4": 10.92656, - "5": 10.93629, - "6": 10.9223, - "7": 10.92716, - "8": 10.92802, - "9": 10.93426, - "10": 10.92691, - "11": 10.92376, - "12": 10.92415, - "13": 10.91326, - "14": 10.91511, - "15": 10.89886, - "16": 10.88689, - "17": 10.8878, - "18": 10.88188, - "19": 10.88572, - "20": 10.8085, - "21": 10.8079, - "22": 10.78818, - "23": 10.78763, - "24": 10.75531, - "25": 10.75377, - "26": 10.74534, - "27": 10.72158, - "28": 10.64806, - "29": 10.62373, - "30": 10.59438, - "31": 10.59725, - "32": 10.57989, - "33": 10.544, - "34": 10.50995, - "35": 10.51437, - "36": 10.49229, - "37": 10.46454, - "38": 10.45962, - "39": 10.43573, - "40": 10.41326, - "41": 10.39006, - "42": 10.37181, - "43": 10.35634, - "44": 10.33171, - "45": 10.33854, - "46": 10.29357, - "47": 10.27812, - "48": 10.2433, - "49": 10.23396, - "50": 10.23414 + "1": 10.9393, + "2": 10.92254, + "3": 10.93003, + "4": 10.92653, + "5": 10.93663, + "6": 10.92231, + "7": 10.92692, + "8": 10.9276, + "9": 10.93447, + "10": 10.92803, + "11": 10.92337, + "12": 10.92395, + "13": 10.9138, + "14": 10.91501, + "15": 10.89836, + "16": 10.88716, + "17": 10.88789, + "18": 10.8814, + "19": 10.88578, + "20": 10.80839, + "21": 10.80736, + "22": 10.78758, + "23": 10.78703, + "24": 10.75415, + "25": 10.7537, + "26": 10.74516, + "27": 10.72174, + "28": 10.64768, + "29": 10.62346, + "30": 10.59487, + "31": 10.59731, + "32": 10.58021, + "33": 10.54459, + "34": 10.51017, + "35": 10.5141, + "36": 10.492, + "37": 10.46445, + "38": 10.45973, + "39": 10.43577, + "40": 10.41313, + "41": 10.39014, + "42": 10.3718, + "43": 10.35627, + "44": 10.33169, + "45": 10.33879, + "46": 10.29367, + "47": 10.27803, + "48": 10.24357, + "49": 10.23406, + "50": 10.23439 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5750.0, - "2": 6129.0, - "3": 5823.0, - "4": 5794.0, - "5": 5914.0, - "6": 5580.0, - "7": 6134.0, - "8": 5917.0, - "9": 5761.0, - "10": 5958.0, - "11": 5805.0, - "12": 5800.0, - "13": 6023.0, - "14": 6095.0, - "15": 5652.0, - "16": 5826.0, - "17": 5902.0, - "18": 5891.0, - "19": 5785.0, - "20": 5685.0, - "21": 5918.0, - "22": 5778.0, - "23": 5868.0, - "24": 5858.0, - "25": 5705.0, - "26": 5769.0, - "27": 5977.0, - "28": 5849.0, - "29": 5911.0, - "30": 5951.0, - "31": 6220.0, - "32": 5937.0, - "33": 5939.0, - "34": 6185.0, - "35": 6164.0, - "36": 6043.0, - "37": 6462.0, - "38": 6229.0, - "39": 6403.0, - "40": 6429.0, - "41": 6671.0, - "42": 6178.0, - "43": 6626.0, - "44": 6265.0, - "45": 7346.0, - "46": 6616.0, - "47": 6836.0, - "48": 7145.0, - "49": 7364.0, - "50": 7278.0 + "1": 5731.0, + "2": 5862.0, + "3": 5904.0, + "4": 5737.0, + "5": 5857.0, + "6": 5686.0, + "7": 5957.0, + "8": 5872.0, + "9": 5831.0, + "10": 5865.0, + "11": 5744.0, + "12": 5673.0, + "13": 5857.0, + "14": 6019.0, + "15": 5625.0, + "16": 5706.0, + "17": 5813.0, + "18": 5764.0, + "19": 5988.0, + "20": 5729.0, + "21": 5754.0, + "22": 5818.0, + "23": 5822.0, + "24": 5984.0, + "25": 5840.0, + "26": 5899.0, + "27": 5941.0, + "28": 5794.0, + "29": 6038.0, + "30": 5982.0, + "31": 6115.0, + "32": 6111.0, + "33": 5966.0, + "34": 6111.0, + "35": 6273.0, + "36": 6002.0, + "37": 6388.0, + "38": 6311.0, + "39": 6345.0, + "40": 6692.0, + "41": 6659.0, + "42": 6076.0, + "43": 6646.0, + "44": 6220.0, + "45": 7071.0, + "46": 6785.0, + "47": 6746.0, + "48": 7209.0, + "49": 7332.0, + "50": 7146.0 } }, "mem-allocated-bytes": { @@ -120,54 +120,54 @@ "values": { "1": 598356992.0, "2": 598359552.0, - "3": 598360064.0, + "3": 598359040.0, "4": 598358016.0, - "5": 598359552.0, + "5": 598359040.0, "6": 598359040.0, - "7": 598355968.0, + "7": 598356992.0, "8": 598358016.0, - "9": 598357504.0, + "9": 598356480.0, "10": 598359040.0, "11": 598358016.0, - "12": 598356992.0, + "12": 598359040.0, "13": 598359040.0, "14": 598355968.0, "15": 598359552.0, "16": 598358016.0, "17": 598359040.0, - "18": 598359040.0, + "18": 598358016.0, "19": 598358528.0, "20": 598358016.0, - "21": 598355968.0, - "22": 598358016.0, - "23": 598358528.0, - "24": 598359040.0, - "25": 598358528.0, + "21": 598356992.0, + "22": 598359040.0, + "23": 598357504.0, + "24": 598358016.0, + "25": 598359552.0, "26": 598358016.0, - "27": 598360064.0, + "27": 598361088.0, "28": 598358016.0, - "29": 598358016.0, + "29": 598359040.0, "30": 598357504.0, "31": 598358016.0, "32": 598358016.0, "33": 598359040.0, "34": 598359040.0, - "35": 598356992.0, + "35": 598358016.0, "36": 598358016.0, "37": 598358528.0, "38": 598358016.0, - "39": 598356992.0, + "39": 598358016.0, "40": 598356992.0, "41": 598355968.0, "42": 598355456.0, "43": 598356480.0, "44": 598355968.0, - "45": 598356992.0, + "45": 598358528.0, "46": 598356992.0, "47": 598356480.0, "48": 598356992.0, "49": 598357504.0, - "50": 598359552.0 + "50": 598358528.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 854438400.0, - "2": 1081444864.0, - "3": 1081444864.0, - "4": 1084059648.0, - "5": 1084059648.0, - "6": 1084059648.0, - "7": 1084059648.0, - "8": 1084059648.0, - "9": 1084059648.0, - "10": 1084059648.0, - "11": 1084059648.0, - "12": 1084059648.0, - "13": 1084059648.0, - "14": 1084059648.0, - "15": 1084059648.0, - "16": 1084059648.0, - "17": 1084489728.0, - "18": 1084489728.0, - "19": 1084489728.0, - "20": 1084489728.0, - "21": 1084489728.0, - "22": 1084489728.0, - "23": 1084489728.0, - "24": 1084489728.0, - "25": 1084489728.0, - "26": 1084489728.0, - "27": 1084489728.0, - "28": 1084489728.0, - "29": 1084489728.0, - "30": 1084489728.0, - "31": 1084489728.0, - "32": 1084489728.0, - "33": 1084489728.0, - "34": 1084489728.0, - "35": 1084489728.0, - "36": 1084489728.0, - "37": 1084489728.0, - "38": 1084489728.0, - "39": 1084489728.0, - "40": 1084489728.0, - "41": 1084489728.0, - "42": 1084489728.0, - "43": 1084489728.0, - "44": 1084489728.0, - "45": 1084489728.0, - "46": 1084489728.0, - "47": 1084489728.0, - "48": 1084489728.0, - "49": 1084489728.0, - "50": 1084489728.0 + "1": 854393344.0, + "2": 1081646080.0, + "3": 1081646080.0, + "4": 1083935744.0, + "5": 1083935744.0, + "6": 1083935744.0, + "7": 1083935744.0, + "8": 1083935744.0, + "9": 1083935744.0, + "10": 1083935744.0, + "11": 1083935744.0, + "12": 1083935744.0, + "13": 1083935744.0, + "14": 1083935744.0, + "15": 1083935744.0, + "16": 1083935744.0, + "17": 1083935744.0, + "18": 1083935744.0, + "19": 1083935744.0, + "20": 1083935744.0, + "21": 1083935744.0, + "22": 1083935744.0, + "23": 1083935744.0, + "24": 1083935744.0, + "25": 1083935744.0, + "26": 1083935744.0, + "27": 1083935744.0, + "28": 1083935744.0, + "29": 1083935744.0, + "30": 1083935744.0, + "31": 1083935744.0, + "32": 1083935744.0, + "33": 1083935744.0, + "34": 1083935744.0, + "35": 1083935744.0, + "36": 1083935744.0, + "37": 1083935744.0, + "38": 1083935744.0, + "39": 1083935744.0, + "40": 1083935744.0, + "41": 1083935744.0, + "42": 1083935744.0, + "43": 1083935744.0, + "44": 1083935744.0, + "45": 1083935744.0, + "46": 1083935744.0, + "47": 1083935744.0, + "48": 1083935744.0, + "49": 1083935744.0, + "50": 1083935744.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.06618, - "3": 0.67089, - "4": 0.65473, - "5": 0.65679, - "6": 0.65585, - "7": 0.65493, - "8": 0.65573, - "9": 0.65578, - "10": 0.65612, - "11": 0.65555, - "12": 0.65414, - "13": 0.65363, - "14": 0.65358, - "15": 0.65375, - "16": 0.65387, - "17": 0.65287, - "18": 0.65388, - "19": 0.65485, - "20": 0.65403, - "21": 0.65913, - "22": 0.6586, - "23": 0.66031, - "24": 0.66152, - "25": 0.66197, - "26": 0.66183, - "27": 0.66179, - "28": 0.66393, - "29": 0.6668, - "30": 0.6651, - "31": 0.66335, - "32": 0.66323, - "33": 0.66088, - "34": 0.66466, - "35": 0.66103, - "36": 0.66076, - "37": 0.65819, - "38": 0.6599, - "39": 0.65814, - "40": 0.65765, - "41": 0.66263, - "42": 0.66244, - "43": 0.662, - "44": 0.66327, - "45": 0.6525, - "46": 0.64778, - "47": 0.64883, - "48": 0.64761, - "49": 0.64897, - "50": 0.64786 + "2": 8.2232, + "3": 0.71083, + "4": 0.68413, + "5": 0.70032, + "6": 0.69143, + "7": 0.69046, + "8": 0.68878, + "9": 0.69004, + "10": 0.69522, + "11": 0.68699, + "12": 0.6909, + "13": 0.69575, + "14": 0.6896, + "15": 0.68723, + "16": 0.68896, + "17": 0.68933, + "18": 0.68784, + "19": 0.68828, + "20": 0.69013, + "21": 0.68703, + "22": 0.6907, + "23": 0.69237, + "24": 0.68963, + "25": 0.69097, + "26": 0.67361, + "27": 0.67299, + "28": 0.67885, + "29": 0.67313, + "30": 0.67193, + "31": 0.6726, + "32": 0.67344, + "33": 0.67275, + "34": 0.67318, + "35": 0.6738, + "36": 0.67398, + "37": 0.67236, + "38": 0.67688, + "39": 0.68487, + "40": 0.6727, + "41": 0.67219, + "42": 0.67599, + "43": 0.68893, + "44": 0.68822, + "45": 0.68432, + "46": 0.6842, + "47": 0.68449, + "48": 0.68494, + "49": 0.68648, + "50": 0.68357 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json index 3b277b10c0a..81282461fb1 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89844, - "2": 10.90039, - "3": 10.89837, - "4": 10.89688, - "5": 10.89885, - "6": 10.88618, - "7": 10.88978, - "8": 10.89165, - "9": 10.89445, - "10": 10.89776, - "11": 10.887, - "12": 10.89063, - "13": 10.88395, - "14": 10.87663, - "15": 10.86762, - "16": 10.85456, - "17": 10.85541, - "18": 10.83795, - "19": 10.85342, - "20": 10.78888, - "21": 10.77633, - "22": 10.77318, - "23": 10.76231, - "24": 10.73158, - "25": 10.72373, - "26": 10.72212, - "27": 10.701, - "28": 10.64017, - "29": 10.60986, - "30": 10.59138, - "31": 10.57684, - "32": 10.55902, - "33": 10.52555, - "34": 10.5102, - "35": 10.49812, - "36": 10.48247, - "37": 10.4528, - "38": 10.46112, - "39": 10.42931, - "40": 10.41237, - "41": 10.39529, - "42": 10.36917, - "43": 10.34961, - "44": 10.32146, - "45": 10.33362, - "46": 10.29354, - "47": 10.28045, - "48": 10.2359, - "49": 10.24617, - "50": 10.23514 + "1": 10.89802, + "2": 10.90021, + "3": 10.89823, + "4": 10.89669, + "5": 10.89951, + "6": 10.88562, + "7": 10.89028, + "8": 10.89225, + "9": 10.89399, + "10": 10.89786, + "11": 10.88695, + "12": 10.88994, + "13": 10.88377, + "14": 10.87719, + "15": 10.8672, + "16": 10.8546, + "17": 10.8549, + "18": 10.83796, + "19": 10.85411, + "20": 10.78881, + "21": 10.7772, + "22": 10.77399, + "23": 10.76223, + "24": 10.73066, + "25": 10.72292, + "26": 10.72271, + "27": 10.70096, + "28": 10.64012, + "29": 10.61024, + "30": 10.59081, + "31": 10.57683, + "32": 10.55893, + "33": 10.52562, + "34": 10.51045, + "35": 10.49804, + "36": 10.48157, + "37": 10.45275, + "38": 10.46063, + "39": 10.42883, + "40": 10.412, + "41": 10.39572, + "42": 10.3692, + "43": 10.34979, + "44": 10.32155, + "45": 10.33299, + "46": 10.29376, + "47": 10.2807, + "48": 10.23603, + "49": 10.24619, + "50": 10.23463 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5981.0, - "2": 5941.0, - "3": 6032.0, - "4": 6141.0, - "5": 5994.0, - "6": 5669.0, - "7": 6099.0, - "8": 5999.0, - "9": 6071.0, - "10": 6120.0, - "11": 5905.0, - "12": 5635.0, - "13": 6149.0, - "14": 6139.0, - "15": 5944.0, - "16": 5941.0, - "17": 6192.0, - "18": 6036.0, - "19": 6011.0, - "20": 5807.0, - "21": 5807.0, - "22": 5888.0, - "23": 5911.0, - "24": 6044.0, - "25": 5619.0, - "26": 5999.0, - "27": 6001.0, - "28": 5983.0, - "29": 6083.0, - "30": 6020.0, - "31": 6002.0, - "32": 6149.0, - "33": 5987.0, - "34": 6100.0, - "35": 6255.0, - "36": 6013.0, - "37": 6476.0, - "38": 6115.0, - "39": 6474.0, - "40": 6498.0, - "41": 6621.0, - "42": 6077.0, - "43": 6920.0, - "44": 6484.0, - "45": 7274.0, - "46": 6778.0, - "47": 7035.0, - "48": 7388.0, - "49": 7581.0, - "50": 7291.0 + "1": 5949.0, + "2": 5859.0, + "3": 6058.0, + "4": 6009.0, + "5": 6077.0, + "6": 5629.0, + "7": 6046.0, + "8": 6056.0, + "9": 6076.0, + "10": 6057.0, + "11": 6040.0, + "12": 5768.0, + "13": 6176.0, + "14": 6151.0, + "15": 5912.0, + "16": 5989.0, + "17": 6088.0, + "18": 5925.0, + "19": 6083.0, + "20": 5820.0, + "21": 5694.0, + "22": 5956.0, + "23": 5936.0, + "24": 6145.0, + "25": 5845.0, + "26": 5951.0, + "27": 5898.0, + "28": 6086.0, + "29": 5955.0, + "30": 5874.0, + "31": 6090.0, + "32": 5974.0, + "33": 5926.0, + "34": 6205.0, + "35": 6237.0, + "36": 6061.0, + "37": 6564.0, + "38": 6296.0, + "39": 6436.0, + "40": 6544.0, + "41": 6819.0, + "42": 6065.0, + "43": 6763.0, + "44": 6546.0, + "45": 7297.0, + "46": 6821.0, + "47": 6931.0, + "48": 7259.0, + "49": 7526.0, + "50": 7287.0 } }, "mem-allocated-bytes": { @@ -121,49 +121,49 @@ "1": 627719168.0, "2": 627717120.0, "3": 627718144.0, - "4": 627718656.0, + "4": 627719168.0, "5": 627717632.0, - "6": 627718656.0, - "7": 627718144.0, - "8": 627718144.0, - "9": 627719168.0, + "6": 627717632.0, + "7": 627719168.0, + "8": 627717120.0, + "9": 627718144.0, "10": 627717120.0, "11": 627719680.0, "12": 627718144.0, - "13": 627717632.0, + "13": 627717120.0, "14": 627720192.0, "15": 627718144.0, - "16": 627720704.0, + "16": 627719680.0, "17": 627720192.0, - "18": 627718144.0, + "18": 627719168.0, "19": 627718656.0, - "20": 627718656.0, - "21": 627717120.0, + "20": 627718144.0, + "21": 627718144.0, "22": 627718144.0, - "23": 627720192.0, + "23": 627719168.0, "24": 627716608.0, "25": 627718144.0, - "26": 627720704.0, + "26": 627720192.0, "27": 627719168.0, "28": 627719168.0, - "29": 627719680.0, + "29": 627719168.0, "30": 627719168.0, "31": 627718144.0, "32": 627720192.0, "33": 627719168.0, "34": 627719168.0, - "35": 627717632.0, - "36": 627719168.0, + "35": 627718144.0, + "36": 627718144.0, "37": 627719168.0, "38": 627719680.0, "39": 627717632.0, - "40": 627719168.0, - "41": 627719168.0, + "40": 627718144.0, + "41": 627720192.0, "42": 627718144.0, - "43": 627719680.0, - "44": 627719680.0, - "45": 627719680.0, - "46": 627720192.0, + "43": 627718656.0, + "44": 627719168.0, + "45": 627718656.0, + "46": 627720704.0, "47": 627720704.0, "48": 627718144.0, "49": 627721216.0, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 878641664.0, - "2": 1112412160.0, - "3": 1112412160.0, - "4": 1113359360.0, - "5": 1113843712.0, - "6": 1113843712.0, - "7": 1113843712.0, - "8": 1113843712.0, - "9": 1113843712.0, - "10": 1113843712.0, - "11": 1113843712.0, - "12": 1113843712.0, - "13": 1113843712.0, - "14": 1113843712.0, - "15": 1113843712.0, - "16": 1113843712.0, - "17": 1113843712.0, - "18": 1113843712.0, - "19": 1113843712.0, - "20": 1113843712.0, - "21": 1113843712.0, - "22": 1113843712.0, - "23": 1113843712.0, - "24": 1113843712.0, - "25": 1113843712.0, - "26": 1113843712.0, - "27": 1113843712.0, - "28": 1113843712.0, - "29": 1113843712.0, - "30": 1113843712.0, - "31": 1113843712.0, - "32": 1113843712.0, - "33": 1113843712.0, - "34": 1113843712.0, - "35": 1113843712.0, - "36": 1113843712.0, - "37": 1113843712.0, - "38": 1113843712.0, - "39": 1113843712.0, - "40": 1113843712.0, - "41": 1113843712.0, - "42": 1113843712.0, - "43": 1113843712.0, - "44": 1113843712.0, - "45": 1113843712.0, - "46": 1113843712.0, - "47": 1113843712.0, - "48": 1113843712.0, - "49": 1113843712.0, - "50": 1113843712.0 + "1": 880758784.0, + "2": 1113250816.0, + "3": 1113250816.0, + "4": 1114362880.0, + "5": 1114579968.0, + "6": 1114579968.0, + "7": 1114579968.0, + "8": 1114579968.0, + "9": 1114579968.0, + "10": 1114579968.0, + "11": 1114579968.0, + "12": 1114579968.0, + "13": 1114579968.0, + "14": 1114579968.0, + "15": 1114579968.0, + "16": 1114579968.0, + "17": 1114579968.0, + "18": 1114579968.0, + "19": 1114579968.0, + "20": 1114579968.0, + "21": 1114579968.0, + "22": 1114579968.0, + "23": 1114579968.0, + "24": 1114579968.0, + "25": 1114579968.0, + "26": 1114579968.0, + "27": 1114579968.0, + "28": 1114579968.0, + "29": 1114579968.0, + "30": 1114579968.0, + "31": 1114579968.0, + "32": 1114579968.0, + "33": 1114579968.0, + "34": 1114937344.0, + "35": 1114937344.0, + "36": 1114937344.0, + "37": 1114937344.0, + "38": 1114937344.0, + "39": 1114937344.0, + "40": 1114937344.0, + "41": 1114937344.0, + "42": 1114937344.0, + "43": 1114937344.0, + "44": 1114937344.0, + "45": 1114937344.0, + "46": 1114937344.0, + "47": 1114937344.0, + "48": 1114937344.0, + "49": 1114937344.0, + "50": 1114937344.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.9557, - "3": 0.82918, - "4": 0.80126, - "5": 0.80359, - "6": 0.80181, - "7": 0.80171, - "8": 0.79822, - "9": 0.79672, - "10": 0.79593, - "11": 0.79799, - "12": 0.80384, - "13": 0.79678, - "14": 0.79858, - "15": 0.79967, - "16": 0.79977, - "17": 0.80141, - "18": 0.79485, - "19": 0.79587, - "20": 0.79661, - "21": 0.80011, - "22": 0.79718, - "23": 0.79688, - "24": 0.79645, - "25": 0.79699, - "26": 0.79975, - "27": 0.80233, - "28": 0.79444, - "29": 0.79372, - "30": 0.79302, - "31": 0.79815, - "32": 0.79752, - "33": 0.79102, - "34": 0.79504, - "35": 0.80282, - "36": 0.80329, - "37": 0.79424, - "38": 0.79725, - "39": 0.79987, - "40": 0.79889, - "41": 0.79755, - "42": 0.79522, - "43": 0.79931, - "44": 0.79859, - "45": 0.79674, - "46": 0.79738, - "47": 0.79179, - "48": 0.79105, - "49": 0.79449, - "50": 0.79649 + "2": 16.07492, + "3": 0.76038, + "4": 0.69701, + "5": 0.68937, + "6": 0.69874, + "7": 0.69551, + "8": 0.69238, + "9": 0.69737, + "10": 0.6972, + "11": 0.69552, + "12": 0.69936, + "13": 0.69942, + "14": 0.69825, + "15": 0.70175, + "16": 0.68838, + "17": 0.69107, + "18": 0.68787, + "19": 0.68382, + "20": 0.68793, + "21": 0.68593, + "22": 0.69095, + "23": 0.68726, + "24": 0.686, + "25": 0.69276, + "26": 0.68751, + "27": 0.69232, + "28": 0.68163, + "29": 0.69196, + "30": 0.69004, + "31": 0.68489, + "32": 0.69228, + "33": 0.6899, + "34": 0.69216, + "35": 0.69442, + "36": 0.69048, + "37": 0.68198, + "38": 0.68983, + "39": 0.6879, + "40": 0.69424, + "41": 0.69521, + "42": 0.69076, + "43": 0.69784, + "44": 0.69188, + "45": 0.68677, + "46": 0.69258, + "47": 0.70212, + "48": 0.69426, + "49": 0.69632, + "50": 0.69736 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json index fde3628e399..bbe6d1ade4c 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93494, - "2": 10.92216, - "3": 10.9323, - "4": 10.93809, - "5": 10.93865, - "6": 10.9346, - "7": 10.93441, - "8": 10.93163, - "9": 10.93122, - "10": 10.93518, - "11": 10.92054, - "12": 10.92608, - "13": 10.90642, - "14": 10.90507, - "15": 10.909, - "16": 10.88742, - "17": 10.88342, - "18": 10.88483, - "19": 10.87524, - "20": 10.81739, - "21": 10.8076, - "22": 10.79095, - "23": 10.7958, - "24": 10.76742, - "25": 10.76749, - "26": 10.74956, - "27": 10.73015, - "28": 10.6667, - "29": 10.63232, - "30": 10.60781, - "31": 10.61094, - "32": 10.5864, - "33": 10.56805, - "34": 10.52557, - "35": 10.53216, - "36": 10.50614, - "37": 10.48124, - "38": 10.48137, - "39": 10.45046, - "40": 10.42825, - "41": 10.41243, - "42": 10.39717, - "43": 10.3757, - "44": 10.3515, - "45": 10.35266, - "46": 10.32123, - "47": 10.3062, - "48": 10.26422, - "49": 10.25789, - "50": 10.25794 + "1": 10.93497, + "2": 10.92175, + "3": 10.93235, + "4": 10.9375, + "5": 10.93907, + "6": 10.93498, + "7": 10.93453, + "8": 10.93082, + "9": 10.93185, + "10": 10.93498, + "11": 10.92081, + "12": 10.92683, + "13": 10.90659, + "14": 10.90486, + "15": 10.9094, + "16": 10.88686, + "17": 10.88327, + "18": 10.88527, + "19": 10.87588, + "20": 10.81635, + "21": 10.80735, + "22": 10.79173, + "23": 10.79525, + "24": 10.76741, + "25": 10.76686, + "26": 10.74909, + "27": 10.73074, + "28": 10.66624, + "29": 10.63199, + "30": 10.60767, + "31": 10.61074, + "32": 10.58623, + "33": 10.56862, + "34": 10.52586, + "35": 10.53161, + "36": 10.5062, + "37": 10.48115, + "38": 10.4817, + "39": 10.45107, + "40": 10.42809, + "41": 10.41226, + "42": 10.39693, + "43": 10.37553, + "44": 10.35212, + "45": 10.35281, + "46": 10.32101, + "47": 10.30626, + "48": 10.26415, + "49": 10.25805, + "50": 10.25789 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5987.0, - "2": 5977.0, - "3": 6039.0, - "4": 5938.0, - "5": 5893.0, - "6": 5729.0, - "7": 6093.0, - "8": 6012.0, - "9": 6061.0, - "10": 6204.0, - "11": 5930.0, - "12": 5870.0, - "13": 6107.0, - "14": 6074.0, - "15": 5913.0, - "16": 5867.0, - "17": 6269.0, - "18": 5830.0, - "19": 5849.0, - "20": 5813.0, - "21": 6148.0, - "22": 5882.0, - "23": 5984.0, - "24": 5934.0, - "25": 5886.0, - "26": 5894.0, - "27": 6105.0, - "28": 6007.0, - "29": 6168.0, - "30": 5987.0, - "31": 6024.0, - "32": 6208.0, - "33": 5977.0, - "34": 6163.0, - "35": 6319.0, - "36": 6098.0, - "37": 6401.0, - "38": 6289.0, - "39": 6429.0, - "40": 6665.0, - "41": 6804.0, - "42": 6160.0, - "43": 6544.0, - "44": 6378.0, - "45": 6863.0, - "46": 6693.0, - "47": 6960.0, - "48": 6986.0, - "49": 7337.0, - "50": 7059.0 + "1": 5856.0, + "2": 5991.0, + "3": 6013.0, + "4": 6135.0, + "5": 5999.0, + "6": 5719.0, + "7": 6256.0, + "8": 6017.0, + "9": 6094.0, + "10": 5941.0, + "11": 6021.0, + "12": 5790.0, + "13": 6108.0, + "14": 6116.0, + "15": 5881.0, + "16": 6019.0, + "17": 6028.0, + "18": 5941.0, + "19": 6094.0, + "20": 5835.0, + "21": 5923.0, + "22": 5942.0, + "23": 5903.0, + "24": 5962.0, + "25": 5788.0, + "26": 6009.0, + "27": 6060.0, + "28": 6069.0, + "29": 6211.0, + "30": 6017.0, + "31": 5972.0, + "32": 6087.0, + "33": 5956.0, + "34": 6038.0, + "35": 6424.0, + "36": 6113.0, + "37": 6390.0, + "38": 6259.0, + "39": 6430.0, + "40": 6691.0, + "41": 6741.0, + "42": 6045.0, + "43": 6576.0, + "44": 6446.0, + "45": 6940.0, + "46": 6925.0, + "47": 6897.0, + "48": 7075.0, + "49": 7472.0, + "50": 7057.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 627719168.0, - "2": 627717120.0, - "3": 627719168.0, - "4": 627717120.0, - "5": 627718144.0, - "6": 627719168.0, - "7": 627719168.0, - "8": 627717632.0, - "9": 627718656.0, - "10": 627719168.0, - "11": 627718656.0, - "12": 627717120.0, - "13": 627718144.0, - "14": 627719168.0, - "15": 627716096.0, - "16": 627717632.0, - "17": 627718144.0, - "18": 627719680.0, - "19": 627718144.0, - "20": 627718144.0, - "21": 627717632.0, - "22": 627718144.0, - "23": 627718656.0, - "24": 627718144.0, - "25": 627719168.0, - "26": 627717632.0, - "27": 627718656.0, - "28": 627719680.0, - "29": 627718144.0, - "30": 627718656.0, - "31": 627718656.0, - "32": 627718144.0, - "33": 627716096.0, - "34": 627719680.0, - "35": 627718144.0, - "36": 627719680.0, - "37": 627716608.0, - "38": 627717120.0, - "39": 627717632.0, - "40": 627718144.0, - "41": 627717120.0, - "42": 627717632.0, - "43": 627717120.0, - "44": 627717120.0, - "45": 627716608.0, - "46": 627718144.0, - "47": 627717120.0, - "48": 627718144.0, - "49": 627718144.0, - "50": 627717120.0 + "1": 628506624.0, + "2": 628504576.0, + "3": 628505600.0, + "4": 628503552.0, + "5": 628504576.0, + "6": 628504576.0, + "7": 628505600.0, + "8": 628504064.0, + "9": 628505088.0, + "10": 628505600.0, + "11": 628504576.0, + "12": 628503552.0, + "13": 628504064.0, + "14": 628504576.0, + "15": 628504576.0, + "16": 628503552.0, + "17": 628504064.0, + "18": 628506112.0, + "19": 628504576.0, + "20": 628504576.0, + "21": 628505600.0, + "22": 628504576.0, + "23": 628504576.0, + "24": 628505600.0, + "25": 628505600.0, + "26": 628505088.0, + "27": 628504064.0, + "28": 628506112.0, + "29": 628504576.0, + "30": 628504576.0, + "31": 628504576.0, + "32": 628505600.0, + "33": 628501504.0, + "34": 628506112.0, + "35": 628504576.0, + "36": 628504576.0, + "37": 628503040.0, + "38": 628503552.0, + "39": 628503552.0, + "40": 628505088.0, + "41": 628503552.0, + "42": 628503040.0, + "43": 628503552.0, + "44": 628503552.0, + "45": 628503552.0, + "46": 628503552.0, + "47": 628503552.0, + "48": 628505600.0, + "49": 628504576.0, + "50": 628503552.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 880525312.0, - "2": 1112519168.0, - "3": 1112519168.0, - "4": 1112519168.0, - "5": 1112898048.0, - "6": 1112898048.0, - "7": 1112898048.0, - "8": 1113367552.0, - "9": 1113367552.0, - "10": 1113367552.0, - "11": 1113367552.0, - "12": 1113367552.0, - "13": 1113367552.0, - "14": 1113367552.0, - "15": 1113367552.0, - "16": 1113367552.0, - "17": 1113367552.0, - "18": 1113367552.0, - "19": 1113620992.0, - "20": 1114435584.0, - "21": 1114435584.0, - "22": 1114435584.0, - "23": 1114435584.0, - "24": 1114435584.0, - "25": 1114435584.0, - "26": 1114435584.0, - "27": 1114435584.0, - "28": 1114435584.0, - "29": 1114435584.0, - "30": 1114435584.0, - "31": 1114435584.0, - "32": 1114435584.0, - "33": 1114435584.0, - "34": 1114435584.0, - "35": 1114435584.0, - "36": 1114435584.0, - "37": 1114435584.0, - "38": 1114435584.0, - "39": 1114435584.0, - "40": 1114435584.0, - "41": 1114435584.0, - "42": 1114435584.0, - "43": 1114435584.0, - "44": 1114435584.0, - "45": 1114435584.0, - "46": 1114435584.0, - "47": 1114435584.0, - "48": 1114435584.0, - "49": 1114435584.0, - "50": 1114435584.0 + "1": 879522304.0, + "2": 1113401856.0, + "3": 1113401856.0, + "4": 1113401856.0, + "5": 1113401856.0, + "6": 1113401856.0, + "7": 1113401856.0, + "8": 1113740288.0, + "9": 1113740288.0, + "10": 1113774592.0, + "11": 1113774592.0, + "12": 1113774592.0, + "13": 1113774592.0, + "14": 1113774592.0, + "15": 1113774592.0, + "16": 1113774592.0, + "17": 1113774592.0, + "18": 1113774592.0, + "19": 1114180096.0, + "20": 1114966528.0, + "21": 1114966528.0, + "22": 1114966528.0, + "23": 1114966528.0, + "24": 1114966528.0, + "25": 1114966528.0, + "26": 1114966528.0, + "27": 1114966528.0, + "28": 1114966528.0, + "29": 1114966528.0, + "30": 1114966528.0, + "31": 1114966528.0, + "32": 1114966528.0, + "33": 1114966528.0, + "34": 1114966528.0, + "35": 1114966528.0, + "36": 1114966528.0, + "37": 1114966528.0, + "38": 1114966528.0, + "39": 1114966528.0, + "40": 1114966528.0, + "41": 1114966528.0, + "42": 1114966528.0, + "43": 1114966528.0, + "44": 1114966528.0, + "45": 1114966528.0, + "46": 1114966528.0, + "47": 1114966528.0, + "48": 1114966528.0, + "49": 1114966528.0, + "50": 1114966528.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 10.42229, - "3": 0.51168, - "4": 0.47502, - "5": 0.46435, - "6": 0.48087, - "7": 0.45841, - "8": 0.4704, - "9": 0.47354, - "10": 0.45958, - "11": 0.45794, - "12": 0.46302, - "13": 0.45848, - "14": 0.45768, - "15": 0.45761, - "16": 0.45984, - "17": 0.4561, - "18": 0.46959, - "19": 0.45852, - "20": 0.45662, - "21": 0.45902, - "22": 0.45503, - "23": 0.48306, - "24": 0.47417, - "25": 0.46981, - "26": 0.46428, - "27": 0.45825, - "28": 0.45603, - "29": 0.45604, - "30": 0.45647, - "31": 0.4568, - "32": 0.45741, - "33": 0.45659, - "34": 0.46102, - "35": 0.46859, - "36": 0.45671, - "37": 0.45741, - "38": 0.45645, - "39": 0.4608, - "40": 0.46228, - "41": 0.45988, - "42": 0.45745, - "43": 0.45884, - "44": 0.46484, - "45": 0.46091, - "46": 0.456, - "47": 0.45788, - "48": 0.45705, - "49": 0.49234, - "50": 0.48473 + "2": 11.1463, + "3": 0.52648, + "4": 0.49102, + "5": 0.4839, + "6": 0.48894, + "7": 0.47777, + "8": 0.47779, + "9": 0.47592, + "10": 0.48159, + "11": 0.47869, + "12": 0.48789, + "13": 0.47507, + "14": 0.47849, + "15": 0.47654, + "16": 0.47752, + "17": 0.47536, + "18": 0.48521, + "19": 0.4779, + "20": 0.47537, + "21": 0.47836, + "22": 0.47661, + "23": 0.47759, + "24": 0.47753, + "25": 0.47987, + "26": 0.48073, + "27": 0.47498, + "28": 0.47388, + "29": 0.47553, + "30": 0.47423, + "31": 0.48396, + "32": 0.49967, + "33": 0.48812, + "34": 0.47832, + "35": 0.47752, + "36": 0.47875, + "37": 0.48128, + "38": 0.48659, + "39": 0.51202, + "40": 0.49079, + "41": 0.50008, + "42": 0.49118, + "43": 0.47903, + "44": 0.48305, + "45": 0.47792, + "46": 0.48084, + "47": 0.48349, + "48": 0.47504, + "49": 0.48679, + "50": 0.47638 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_a100.json index 29c154228bd..64b94804ab5 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_a100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.9191, - "2": 10.91657, - "3": 10.92612, - "4": 10.91743, - "5": 10.92515, - "6": 10.91455, - "7": 10.91757, - "8": 10.9188, - "9": 10.9198, - "10": 10.92142, - "11": 10.91544, - "12": 10.91366, - "13": 10.89765, - "14": 10.90237, - "15": 10.8841, - "16": 10.88468, - "17": 10.87811, - "18": 10.87305, - "19": 10.86617, - "20": 10.79818, - "21": 10.79896, - "22": 10.78314, - "23": 10.78596, - "24": 10.74431, - "25": 10.76005, - "26": 10.72959, - "27": 10.7235, - "28": 10.64773, - "29": 10.61837, - "30": 10.59978, - "31": 10.60059, - "32": 10.57623, - "33": 10.55868, - "34": 10.51277, - "35": 10.51753, - "36": 10.51378, - "37": 10.47335, - "38": 10.46866, - "39": 10.45387, - "40": 10.43424, - "41": 10.4083, - "42": 10.39229, - "43": 10.37761, - "44": 10.34934, - "45": 10.35892, - "46": 10.31862, - "47": 10.30601, - "48": 10.26806, - "49": 10.25481, - "50": 10.26589 + "1": 10.92007, + "2": 10.91646, + "3": 10.92617, + "4": 10.91763, + "5": 10.92559, + "6": 10.91443, + "7": 10.91661, + "8": 10.91832, + "9": 10.92012, + "10": 10.92144, + "11": 10.91542, + "12": 10.91428, + "13": 10.89766, + "14": 10.90217, + "15": 10.88415, + "16": 10.88446, + "17": 10.87736, + "18": 10.87282, + "19": 10.86559, + "20": 10.79824, + "21": 10.79883, + "22": 10.7828, + "23": 10.7852, + "24": 10.74413, + "25": 10.75902, + "26": 10.7296, + "27": 10.72396, + "28": 10.64747, + "29": 10.61781, + "30": 10.59955, + "31": 10.60032, + "32": 10.57668, + "33": 10.55896, + "34": 10.51264, + "35": 10.51768, + "36": 10.51356, + "37": 10.47325, + "38": 10.46901, + "39": 10.45416, + "40": 10.43408, + "41": 10.40788, + "42": 10.39227, + "43": 10.37741, + "44": 10.34965, + "45": 10.35859, + "46": 10.31887, + "47": 10.30644, + "48": 10.26835, + "49": 10.25488, + "50": 10.26582 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6636.0, - "2": 6475.0, - "3": 6570.0, - "4": 6636.0, - "5": 6399.0, - "6": 6400.0, - "7": 6829.0, - "8": 6547.0, - "9": 6746.0, - "10": 6769.0, - "11": 6595.0, - "12": 6215.0, - "13": 6488.0, - "14": 6605.0, - "15": 6555.0, - "16": 6724.0, - "17": 6683.0, - "18": 6720.0, - "19": 6686.0, - "20": 6446.0, - "21": 6643.0, - "22": 6786.0, - "23": 6769.0, - "24": 6544.0, - "25": 6583.0, - "26": 6630.0, - "27": 6774.0, - "28": 6957.0, - "29": 6987.0, - "30": 6549.0, - "31": 7066.0, - "32": 6858.0, - "33": 6666.0, - "34": 6949.0, - "35": 7007.0, - "36": 6782.0, - "37": 7318.0, - "38": 7191.0, - "39": 7055.0, - "40": 7486.0, - "41": 7440.0, - "42": 6901.0, - "43": 7459.0, - "44": 7117.0, - "45": 8044.0, - "46": 7520.0, - "47": 7744.0, - "48": 8052.0, - "49": 8201.0, - "50": 7942.0 + "1": 6551.0, + "2": 6568.0, + "3": 6658.0, + "4": 6484.0, + "5": 6416.0, + "6": 6449.0, + "7": 6849.0, + "8": 6523.0, + "9": 6714.0, + "10": 6721.0, + "11": 6740.0, + "12": 6438.0, + "13": 6835.0, + "14": 6580.0, + "15": 6483.0, + "16": 6662.0, + "17": 6613.0, + "18": 6616.0, + "19": 6794.0, + "20": 6476.0, + "21": 6779.0, + "22": 6751.0, + "23": 6651.0, + "24": 6595.0, + "25": 6516.0, + "26": 6755.0, + "27": 6669.0, + "28": 6867.0, + "29": 6905.0, + "30": 6609.0, + "31": 7012.0, + "32": 6827.0, + "33": 6818.0, + "34": 6950.0, + "35": 6838.0, + "36": 6743.0, + "37": 7117.0, + "38": 7026.0, + "39": 7200.0, + "40": 7335.0, + "41": 7388.0, + "42": 6842.0, + "43": 7461.0, + "44": 7177.0, + "45": 7972.0, + "46": 7532.0, + "47": 7653.0, + "48": 7939.0, + "49": 8325.0, + "50": 7914.0 } }, "mem-allocated-bytes": { @@ -119,53 +119,53 @@ "step_interval": 1, "values": { "1": 462408192.0, - "2": 462408704.0, - "3": 462409216.0, - "4": 462406144.0, + "2": 462409728.0, + "3": 462408704.0, + "4": 462407680.0, "5": 462408192.0, "6": 462406656.0, - "7": 462409216.0, + "7": 462410240.0, "8": 462409216.0, "9": 462406656.0, - "10": 462407168.0, - "11": 462408704.0, - "12": 462408192.0, - "13": 462409728.0, + "10": 462406656.0, + "11": 462408192.0, + "12": 462407168.0, + "13": 462409216.0, "14": 462409728.0, "15": 462408192.0, - "16": 462408192.0, - "17": 462408192.0, + "16": 462407168.0, + "17": 462409216.0, "18": 462409728.0, "19": 462409216.0, "20": 462409216.0, "21": 462406656.0, "22": 462410240.0, "23": 462408192.0, - "24": 462407680.0, - "25": 462408704.0, + "24": 462406656.0, + "25": 462409728.0, "26": 462406656.0, - "27": 462407680.0, + "27": 462408192.0, "28": 462408192.0, - "29": 462408704.0, - "30": 462408192.0, + "29": 462408192.0, + "30": 462409216.0, "31": 462409216.0, "32": 462408704.0, "33": 462407168.0, "34": 462408192.0, - "35": 462407168.0, + "35": 462406656.0, "36": 462407680.0, "37": 462406656.0, - "38": 462408704.0, - "39": 462408192.0, + "38": 462408192.0, + "39": 462409216.0, "40": 462409216.0, "41": 462408704.0, - "42": 462408192.0, - "43": 462407680.0, - "44": 462406656.0, - "45": 462409216.0, - "46": 462408192.0, - "47": 462408192.0, - "48": 462408704.0, + "42": 462407168.0, + "43": 462408704.0, + "44": 462407680.0, + "45": 462410240.0, + "46": 462408704.0, + "47": 462407680.0, + "48": 462407680.0, "49": 462409728.0, "50": 462409216.0 } @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1032205824.0, - "2": 1196511744.0, - "3": 1196511744.0, - "4": 1196529664.0, - "5": 1196529664.0, - "6": 1196529664.0, - "7": 1196529664.0, - "8": 1196529664.0, - "9": 1196605440.0, - "10": 1196605440.0, - "11": 1196605440.0, - "12": 1196605440.0, - "13": 1196605440.0, - "14": 1196605440.0, - "15": 1196605440.0, - "16": 1196605440.0, - "17": 1196605440.0, - "18": 1196605440.0, - "19": 1196605440.0, - "20": 1196605440.0, - "21": 1196605440.0, - "22": 1196605440.0, - "23": 1196605440.0, - "24": 1196605440.0, - "25": 1196605440.0, - "26": 1196605440.0, - "27": 1196605440.0, - "28": 1196605440.0, - "29": 1196605440.0, - "30": 1196605440.0, - "31": 1196605440.0, - "32": 1196605440.0, - "33": 1197258240.0, - "34": 1197258240.0, - "35": 1197258240.0, - "36": 1197258240.0, - "37": 1197258240.0, - "38": 1197258240.0, - "39": 1197258240.0, - "40": 1197258240.0, - "41": 1197258240.0, - "42": 1197258240.0, - "43": 1197258240.0, - "44": 1197258240.0, - "45": 1197258240.0, - "46": 1197258240.0, - "47": 1197258240.0, - "48": 1197258240.0, - "49": 1197258240.0, - "50": 1197258240.0 + "1": 1032228352.0, + "2": 1196615168.0, + "3": 1196615168.0, + "4": 1196615168.0, + "5": 1196615168.0, + "6": 1196615168.0, + "7": 1196615168.0, + "8": 1196615168.0, + "9": 1196615168.0, + "10": 1196630528.0, + "11": 1196630528.0, + "12": 1196630528.0, + "13": 1196630528.0, + "14": 1196630528.0, + "15": 1196630528.0, + "16": 1196630528.0, + "17": 1196630528.0, + "18": 1196630528.0, + "19": 1196630528.0, + "20": 1196630528.0, + "21": 1196630528.0, + "22": 1196630528.0, + "23": 1196630528.0, + "24": 1196630528.0, + "25": 1196630528.0, + "26": 1196630528.0, + "27": 1196630528.0, + "28": 1196630528.0, + "29": 1196630528.0, + "30": 1196630528.0, + "31": 1196630528.0, + "32": 1196630528.0, + "33": 1197310464.0, + "34": 1197310464.0, + "35": 1197310464.0, + "36": 1197310464.0, + "37": 1197310464.0, + "38": 1197310464.0, + "39": 1197310464.0, + "40": 1197310464.0, + "41": 1197310464.0, + "42": 1197310464.0, + "43": 1197310464.0, + "44": 1197310464.0, + "45": 1197310464.0, + "46": 1197310464.0, + "47": 1197310464.0, + "48": 1197310464.0, + "49": 1197310464.0, + "50": 1197310464.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.49757, - "3": 0.57052, - "4": 0.5478, - "5": 0.54995, - "6": 0.54924, - "7": 0.54865, - "8": 0.54844, - "9": 0.5483, - "10": 0.55003, - "11": 0.54852, - "12": 0.54601, - "13": 0.54499, - "14": 0.54371, - "15": 0.54289, - "16": 0.54282, - "17": 0.54283, - "18": 0.54396, - "19": 0.54176, - "20": 0.54265, - "21": 0.54107, - "22": 0.54253, - "23": 0.54291, - "24": 0.54259, - "25": 0.5424, - "26": 0.54199, - "27": 0.5412, - "28": 0.54336, - "29": 0.54351, - "30": 0.54433, - "31": 0.54187, - "32": 0.54165, - "33": 0.5419, - "34": 0.54248, - "35": 0.54246, - "36": 0.54241, - "37": 0.54076, - "38": 0.54289, - "39": 0.54281, - "40": 0.54911, - "41": 0.5473, - "42": 0.5465, - "43": 0.54349, - "44": 0.54718, - "45": 0.54732, - "46": 0.54664, - "47": 0.54702, - "48": 0.54711, - "49": 0.54685, - "50": 0.54625 + "2": 4.79202, + "3": 0.58202, + "4": 0.56507, + "5": 0.56418, + "6": 0.56387, + "7": 0.56352, + "8": 0.56141, + "9": 0.56188, + "10": 0.56138, + "11": 0.56078, + "12": 0.56237, + "13": 0.56767, + "14": 0.56478, + "15": 0.57437, + "16": 0.57496, + "17": 0.57353, + "18": 0.57496, + "19": 0.56981, + "20": 0.57236, + "21": 0.56716, + "22": 0.58108, + "23": 0.57198, + "24": 0.56837, + "25": 0.56822, + "26": 0.5658, + "27": 0.56356, + "28": 0.56412, + "29": 0.56327, + "30": 0.56356, + "31": 0.56451, + "32": 0.5632, + "33": 0.56544, + "34": 0.56418, + "35": 0.56561, + "36": 0.56634, + "37": 0.56351, + "38": 0.56468, + "39": 0.56533, + "40": 0.56347, + "41": 0.56467, + "42": 0.56419, + "43": 0.56488, + "44": 0.5655, + "45": 0.5637, + "46": 0.56462, + "47": 0.56884, + "48": 0.5645, + "49": 0.56529, + "50": 0.56376 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index 6daec7744d2..5702fb59822 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92051, - "2": 10.92395, - "3": 10.92254, - "4": 10.92049, - "5": 10.91474, - "6": 10.90369, - "7": 10.9112, - "8": 10.91582, - "9": 10.91966, - "10": 10.91909, - "11": 10.90563, - "12": 10.90649, - "13": 10.90489, - "14": 10.89897, - "15": 10.87714, - "16": 10.87001, - "17": 10.8834, - "18": 10.86628, - "19": 10.86686, + "1": 10.92142, + "2": 10.92363, + "3": 10.92242, + "4": 10.92026, + "5": 10.91483, + "6": 10.90385, + "7": 10.91109, + "8": 10.91595, + "9": 10.92047, + "10": 10.91896, + "11": 10.90617, + "12": 10.90579, + "13": 10.90562, + "14": 10.8989, + "15": 10.87723, + "16": 10.86957, + "17": 10.8846, + "18": 10.86603, + "19": 10.86663, "20": 10.80107, "21": 10.79745, - "22": 10.78577, - "23": 10.77879, - "24": 10.74333, - "25": 10.74705, - "26": 10.7275, - "27": 10.70964, - "28": 10.64399, - "29": 10.60607, - "30": 10.59747, - "31": 10.58617, - "32": 10.57896, - "33": 10.54631, - "34": 10.50565, - "35": 10.50708, - "36": 10.49921, - "37": 10.45846, - "38": 10.4669, - "39": 10.43069, - "40": 10.42274, - "41": 10.40014, - "42": 10.38466, - "43": 10.36939, - "44": 10.34001, - "45": 10.34882, - "46": 10.31053, - "47": 10.29582, - "48": 10.2503, - "49": 10.24405, - "50": 10.25168 + "22": 10.78608, + "23": 10.77958, + "24": 10.74282, + "25": 10.74782, + "26": 10.72794, + "27": 10.70948, + "28": 10.64378, + "29": 10.60676, + "30": 10.59805, + "31": 10.58664, + "32": 10.57936, + "33": 10.546, + "34": 10.50679, + "35": 10.50654, + "36": 10.49965, + "37": 10.45837, + "38": 10.46672, + "39": 10.43117, + "40": 10.42314, + "41": 10.4007, + "42": 10.38462, + "43": 10.36964, + "44": 10.34028, + "45": 10.34887, + "46": 10.31013, + "47": 10.29593, + "48": 10.25007, + "49": 10.24404, + "50": 10.25153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6472.0, - "2": 6590.0, - "3": 6715.0, - "4": 6477.0, - "5": 6647.0, - "6": 6417.0, - "7": 6895.0, - "8": 6525.0, - "9": 6635.0, - "10": 6725.0, - "11": 6782.0, - "12": 6345.0, - "13": 6855.0, - "14": 6764.0, - "15": 6484.0, - "16": 6478.0, - "17": 6699.0, - "18": 6486.0, - "19": 6797.0, - "20": 6592.0, - "21": 6504.0, - "22": 6673.0, - "23": 6620.0, - "24": 6525.0, - "25": 6471.0, - "26": 6508.0, - "27": 6578.0, - "28": 6593.0, - "29": 6541.0, - "30": 6573.0, - "31": 6920.0, - "32": 6681.0, - "33": 6825.0, - "34": 7093.0, - "35": 7100.0, - "36": 6669.0, - "37": 7293.0, - "38": 7094.0, - "39": 7224.0, - "40": 7424.0, - "41": 7389.0, - "42": 6727.0, - "43": 7386.0, - "44": 7247.0, - "45": 7896.0, - "46": 7544.0, - "47": 7638.0, - "48": 7890.0, - "49": 8076.0, - "50": 7894.0 + "1": 6494.0, + "2": 6740.0, + "3": 6956.0, + "4": 6747.0, + "5": 6648.0, + "6": 6407.0, + "7": 6764.0, + "8": 6471.0, + "9": 6733.0, + "10": 6750.0, + "11": 6865.0, + "12": 6288.0, + "13": 6782.0, + "14": 6749.0, + "15": 6437.0, + "16": 6530.0, + "17": 6644.0, + "18": 6336.0, + "19": 6734.0, + "20": 6356.0, + "21": 6588.0, + "22": 6652.0, + "23": 6530.0, + "24": 6665.0, + "25": 6702.0, + "26": 6687.0, + "27": 6681.0, + "28": 6510.0, + "29": 6670.0, + "30": 6556.0, + "31": 6782.0, + "32": 6863.0, + "33": 6884.0, + "34": 7126.0, + "35": 7032.0, + "36": 6895.0, + "37": 7186.0, + "38": 6999.0, + "39": 7246.0, + "40": 7364.0, + "41": 7488.0, + "42": 6879.0, + "43": 7620.0, + "44": 7231.0, + "45": 7956.0, + "46": 7554.0, + "47": 7513.0, + "48": 7742.0, + "49": 8157.0, + "50": 7799.0 } }, "mem-allocated-bytes": { @@ -119,55 +119,55 @@ "step_interval": 1, "values": { "1": 491766272.0, - "2": 491768320.0, + "2": 491769344.0, "3": 491768320.0, "4": 491769856.0, - "5": 491770368.0, - "6": 491766784.0, - "7": 491766272.0, - "8": 491766272.0, + "5": 491768320.0, + "6": 491767808.0, + "7": 491765248.0, + "8": 491767296.0, "9": 491768832.0, "10": 491769344.0, "11": 491768832.0, - "12": 491768832.0, - "13": 491767296.0, + "12": 491769856.0, + "13": 491766784.0, "14": 491769344.0, "15": 491769344.0, "16": 491767808.0, "17": 491767296.0, - "18": 491768832.0, + "18": 491769344.0, "19": 491769344.0, - "20": 491769344.0, + "20": 491768832.0, "21": 491765760.0, "22": 491770880.0, "23": 491767808.0, - "24": 491768320.0, + "24": 491768832.0, "25": 491769344.0, - "26": 491767296.0, + "26": 491766784.0, "27": 491768320.0, "28": 491767808.0, "29": 491766272.0, - "30": 491769344.0, - "31": 491770880.0, - "32": 491768320.0, - "33": 491767808.0, - "34": 491766784.0, - "35": 491768320.0, + "30": 491768832.0, + "31": 491768832.0, + "32": 491769856.0, + "33": 491768320.0, + "34": 491768832.0, + "35": 491769344.0, "36": 491770368.0, - "37": 491769344.0, + "37": 491768320.0, "38": 491769856.0, "39": 491767808.0, - "40": 491767808.0, - "41": 491770880.0, - "42": 491771392.0, - "43": 491768832.0, + "40": 491767296.0, + "41": 491769856.0, + "42": 491770368.0, + "43": 491767808.0, "44": 491768832.0, - "45": 491769344.0, + "45": 491770368.0, "46": 491771392.0, "47": 491769344.0, "48": 491770368.0, "49": 491770368.0, - "50": 491769856.0 + "50": 491768832.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1059454976.0, - "2": 1225387520.0, - "3": 1225387520.0, - "4": 1225387520.0, - "5": 1225387520.0, - "6": 1226523136.0, - "7": 1226523136.0, - "8": 1226523136.0, - "9": 1226523136.0, - "10": 1226523136.0, - "11": 1226523136.0, - "12": 1226523136.0, - "13": 1226523136.0, - "14": 1226523136.0, - "15": 1226523136.0, - "16": 1226523136.0, - "17": 1226523136.0, - "18": 1226523136.0, - "19": 1226523136.0, - "20": 1226523136.0, - "21": 1226523136.0, - "22": 1226523136.0, - "23": 1226523136.0, - "24": 1226523136.0, - "25": 1226523136.0, - "26": 1226523136.0, - "27": 1226523136.0, - "28": 1226523136.0, - "29": 1226523136.0, - "30": 1226523136.0, - "31": 1226523136.0, - "32": 1226523136.0, - "33": 1226523136.0, - "34": 1226523136.0, - "35": 1226523136.0, - "36": 1226523136.0, - "37": 1226523136.0, - "38": 1226523136.0, - "39": 1226523136.0, - "40": 1226523136.0, - "41": 1226523136.0, - "42": 1226523136.0, - "43": 1226523136.0, - "44": 1226523136.0, - "45": 1226523136.0, - "46": 1226523136.0, - "47": 1226523136.0, - "48": 1226523136.0, - "49": 1226523136.0, - "50": 1226523136.0 + "1": 1060007936.0, + "2": 1225918976.0, + "3": 1225918976.0, + "4": 1226195456.0, + "5": 1226195456.0, + "6": 1226195456.0, + "7": 1226195456.0, + "8": 1226195456.0, + "9": 1226195456.0, + "10": 1226195456.0, + "11": 1226195456.0, + "12": 1226195456.0, + "13": 1226195456.0, + "14": 1226288128.0, + "15": 1226288128.0, + "16": 1226288128.0, + "17": 1226288128.0, + "18": 1226288128.0, + "19": 1226288128.0, + "20": 1226370048.0, + "21": 1226370048.0, + "22": 1226370048.0, + "23": 1226370048.0, + "24": 1226370048.0, + "25": 1226370048.0, + "26": 1226370048.0, + "27": 1226370048.0, + "28": 1226370048.0, + "29": 1226370048.0, + "30": 1226370048.0, + "31": 1226370048.0, + "32": 1226370048.0, + "33": 1226413056.0, + "34": 1226413056.0, + "35": 1226587136.0, + "36": 1226587136.0, + "37": 1226587136.0, + "38": 1226587136.0, + "39": 1226587136.0, + "40": 1226587136.0, + "41": 1226592768.0, + "42": 1226592768.0, + "43": 1226592768.0, + "44": 1226592768.0, + "45": 1226592768.0, + "46": 1226592768.0, + "47": 1226592768.0, + "48": 1226592768.0, + "49": 1226592768.0, + "50": 1226592768.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.05534, - "3": 0.69213, - "4": 0.67022, - "5": 0.66557, - "6": 0.66925, - "7": 0.66358, - "8": 0.66502, - "9": 0.66566, - "10": 0.67305, - "11": 0.66335, - "12": 0.6679, - "13": 0.66572, - "14": 0.66473, - "15": 0.66188, - "16": 0.66168, - "17": 0.66422, - "18": 0.67171, - "19": 0.66451, - "20": 0.66985, - "21": 0.66421, - "22": 0.67531, - "23": 0.66253, - "24": 0.66398, - "25": 0.66282, - "26": 0.66325, - "27": 0.66131, - "28": 0.65976, - "29": 0.66042, - "30": 0.664, - "31": 0.66141, - "32": 0.66508, - "33": 0.66238, - "34": 0.66252, - "35": 0.65952, - "36": 0.65926, - "37": 0.66407, - "38": 0.65743, - "39": 0.66028, - "40": 0.66183, - "41": 0.66404, - "42": 0.66235, - "43": 0.66248, - "44": 0.66216, - "45": 0.66334, - "46": 0.66253, - "47": 0.66097, - "48": 0.66018, - "49": 0.66271, - "50": 0.66137 + "2": 11.80426, + "3": 0.60812, + "4": 0.55699, + "5": 0.5606, + "6": 0.56587, + "7": 0.56787, + "8": 0.57008, + "9": 0.56246, + "10": 0.56156, + "11": 0.55493, + "12": 0.56849, + "13": 0.56815, + "14": 0.55347, + "15": 0.55583, + "16": 0.56418, + "17": 0.56483, + "18": 0.56107, + "19": 0.56165, + "20": 0.55104, + "21": 0.55438, + "22": 0.55262, + "23": 0.55885, + "24": 0.5572, + "25": 0.55888, + "26": 0.55756, + "27": 0.55707, + "28": 0.55433, + "29": 0.55322, + "30": 0.56493, + "31": 0.55972, + "32": 0.5601, + "33": 0.55473, + "34": 0.55994, + "35": 0.55595, + "36": 0.55082, + "37": 0.55663, + "38": 0.56424, + "39": 0.55592, + "40": 0.55588, + "41": 0.55818, + "42": 0.5521, + "43": 0.55345, + "44": 0.55588, + "45": 0.55213, + "46": 0.5499, + "47": 0.55069, + "48": 0.5565, + "49": 0.55117, + "50": 0.55276 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_h100.json index 02ab0400a63..7aab062ad3b 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94605, - "2": 10.93984, - "3": 10.94844, - "4": 10.94383, - "5": 10.94536, - "6": 10.94551, - "7": 10.94076, - "8": 10.93525, - "9": 10.93574, - "10": 10.93995, - "11": 10.9232, - "12": 10.92749, - "13": 10.93106, - "14": 10.92607, - "15": 10.90323, - "16": 10.88838, - "17": 10.90185, - "18": 10.89577, - "19": 10.89614, - "20": 10.80805, - "21": 10.8201, - "22": 10.80485, - "23": 10.80528, - "24": 10.76201, - "25": 10.77306, - "26": 10.74874, - "27": 10.73469, - "28": 10.67415, - "29": 10.64362, - "30": 10.6161, - "31": 10.60746, - "32": 10.59793, - "33": 10.56729, - "34": 10.53273, - "35": 10.53545, - "36": 10.52116, - "37": 10.48954, - "38": 10.48719, - "39": 10.46139, - "40": 10.44216, - "41": 10.4234, - "42": 10.41064, - "43": 10.39619, - "44": 10.36242, - "45": 10.37314, - "46": 10.33605, - "47": 10.32291, - "48": 10.28505, - "49": 10.27029, - "50": 10.27674 + "1": 10.94588, + "2": 10.93966, + "3": 10.94811, + "4": 10.94371, + "5": 10.94538, + "6": 10.94555, + "7": 10.94068, + "8": 10.93521, + "9": 10.93557, + "10": 10.93962, + "11": 10.92315, + "12": 10.92793, + "13": 10.93059, + "14": 10.92558, + "15": 10.90423, + "16": 10.88742, + "17": 10.90114, + "18": 10.89548, + "19": 10.89627, + "20": 10.80893, + "21": 10.82025, + "22": 10.80528, + "23": 10.80526, + "24": 10.76273, + "25": 10.77254, + "26": 10.74899, + "27": 10.73484, + "28": 10.67376, + "29": 10.64442, + "30": 10.61672, + "31": 10.60725, + "32": 10.59762, + "33": 10.56673, + "34": 10.53272, + "35": 10.53538, + "36": 10.52082, + "37": 10.48908, + "38": 10.48669, + "39": 10.46215, + "40": 10.44264, + "41": 10.42395, + "42": 10.4105, + "43": 10.3965, + "44": 10.36255, + "45": 10.373, + "46": 10.33641, + "47": 10.32243, + "48": 10.28518, + "49": 10.27081, + "50": 10.27664 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6735.0, - "2": 6805.0, - "3": 6618.0, - "4": 6774.0, - "5": 6712.0, - "6": 6531.0, - "7": 6946.0, - "8": 6655.0, - "9": 6914.0, - "10": 6647.0, - "11": 6828.0, - "12": 6387.0, - "13": 6821.0, - "14": 6724.0, - "15": 6698.0, - "16": 6520.0, - "17": 6900.0, - "18": 6532.0, - "19": 6843.0, - "20": 6411.0, - "21": 6559.0, - "22": 6525.0, - "23": 6655.0, - "24": 6550.0, - "25": 6633.0, - "26": 6637.0, - "27": 6765.0, - "28": 6593.0, - "29": 6801.0, - "30": 6515.0, - "31": 6821.0, - "32": 6859.0, - "33": 6841.0, - "34": 6861.0, - "35": 7014.0, - "36": 6780.0, - "37": 7226.0, - "38": 7060.0, - "39": 7169.0, - "40": 7319.0, - "41": 7417.0, - "42": 6748.0, - "43": 7596.0, - "44": 7102.0, - "45": 7827.0, - "46": 7407.0, - "47": 7564.0, - "48": 7726.0, - "49": 8052.0, - "50": 7755.0 + "1": 6781.0, + "2": 6861.0, + "3": 6659.0, + "4": 6776.0, + "5": 6623.0, + "6": 6603.0, + "7": 6865.0, + "8": 6613.0, + "9": 6778.0, + "10": 6831.0, + "11": 6880.0, + "12": 6623.0, + "13": 6913.0, + "14": 6919.0, + "15": 6457.0, + "16": 6555.0, + "17": 6782.0, + "18": 6633.0, + "19": 6858.0, + "20": 6320.0, + "21": 6561.0, + "22": 6510.0, + "23": 6652.0, + "24": 6702.0, + "25": 6433.0, + "26": 6613.0, + "27": 6579.0, + "28": 6640.0, + "29": 6909.0, + "30": 6622.0, + "31": 6744.0, + "32": 6820.0, + "33": 6721.0, + "34": 6972.0, + "35": 6901.0, + "36": 6868.0, + "37": 7481.0, + "38": 6965.0, + "39": 7181.0, + "40": 7424.0, + "41": 7386.0, + "42": 6905.0, + "43": 7306.0, + "44": 6939.0, + "45": 7642.0, + "46": 7417.0, + "47": 7607.0, + "48": 7685.0, + "49": 8041.0, + "50": 7703.0 } }, "mem-allocated-bytes": { @@ -118,53 +118,53 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 491769344.0, - "2": 491769344.0, - "3": 491768320.0, - "4": 491768832.0, + "1": 491769856.0, + "2": 491769856.0, + "3": 491768832.0, + "4": 491767296.0, "5": 491768832.0, "6": 491769856.0, - "7": 491767808.0, - "8": 491768320.0, - "9": 491767296.0, + "7": 491766784.0, + "8": 491767296.0, + "9": 491768320.0, "10": 491767296.0, - "11": 491767808.0, + "11": 491766784.0, "12": 491768320.0, - "13": 491767296.0, + "13": 491769344.0, "14": 491766784.0, - "15": 491769856.0, - "16": 491769344.0, + "15": 491770880.0, + "16": 491768320.0, "17": 491767808.0, "18": 491768320.0, "19": 491767296.0, "20": 491766784.0, - "21": 491767296.0, - "22": 491768320.0, + "21": 491768832.0, + "22": 491768832.0, "23": 491768320.0, "24": 491767296.0, "25": 491766272.0, - "26": 491767296.0, - "27": 491767808.0, + "26": 491765760.0, + "27": 491766784.0, "28": 491768832.0, "29": 491769344.0, - "30": 491768320.0, - "31": 491767296.0, - "32": 491769344.0, - "33": 491768320.0, - "34": 491766784.0, + "30": 491769344.0, + "31": 491766272.0, + "32": 491768832.0, + "33": 491767808.0, + "34": 491767808.0, "35": 491766784.0, - "36": 491766272.0, + "36": 491766784.0, "37": 491767808.0, - "38": 491768832.0, + "38": 491768320.0, "39": 491766784.0, "40": 491766784.0, "41": 491767808.0, "42": 491766784.0, "43": 491765760.0, "44": 491768320.0, - "45": 491768320.0, - "46": 491768832.0, - "47": 491768320.0, + "45": 491769856.0, + "46": 491769856.0, + "47": 491768832.0, "48": 491766784.0, "49": 491769856.0, "50": 491767296.0 @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1059374080.0, - "2": 1225970688.0, - "3": 1225970688.0, - "4": 1225970688.0, - "5": 1225970688.0, - "6": 1225970688.0, - "7": 1225970688.0, - "8": 1225970688.0, - "9": 1225970688.0, - "10": 1225970688.0, - "11": 1225970688.0, - "12": 1225970688.0, - "13": 1225970688.0, - "14": 1225970688.0, - "15": 1225970688.0, - "16": 1225970688.0, - "17": 1225970688.0, - "18": 1225970688.0, - "19": 1225970688.0, - "20": 1225970688.0, - "21": 1225970688.0, - "22": 1225970688.0, - "23": 1225970688.0, - "24": 1225970688.0, - "25": 1225970688.0, - "26": 1225970688.0, - "27": 1225970688.0, - "28": 1225970688.0, - "29": 1225970688.0, - "30": 1225970688.0, - "31": 1225970688.0, - "32": 1225970688.0, - "33": 1225970688.0, - "34": 1225970688.0, - "35": 1225970688.0, - "36": 1225970688.0, - "37": 1225970688.0, - "38": 1225970688.0, - "39": 1225970688.0, - "40": 1225970688.0, - "41": 1225970688.0, - "42": 1225970688.0, - "43": 1225970688.0, - "44": 1225970688.0, - "45": 1225970688.0, - "46": 1225970688.0, - "47": 1225970688.0, - "48": 1225970688.0, - "49": 1225970688.0, - "50": 1225970688.0 + "1": 1060344832.0, + "2": 1225047040.0, + "3": 1225791488.0, + "4": 1225791488.0, + "5": 1225791488.0, + "6": 1225791488.0, + "7": 1225791488.0, + "8": 1225791488.0, + "9": 1225831936.0, + "10": 1225831936.0, + "11": 1225831936.0, + "12": 1225831936.0, + "13": 1225831936.0, + "14": 1225831936.0, + "15": 1225831936.0, + "16": 1225831936.0, + "17": 1225831936.0, + "18": 1225831936.0, + "19": 1225831936.0, + "20": 1225831936.0, + "21": 1225831936.0, + "22": 1225831936.0, + "23": 1225831936.0, + "24": 1225831936.0, + "25": 1225831936.0, + "26": 1225831936.0, + "27": 1226185728.0, + "28": 1226185728.0, + "29": 1226185728.0, + "30": 1226185728.0, + "31": 1226185728.0, + "32": 1226185728.0, + "33": 1226185728.0, + "34": 1226185728.0, + "35": 1226185728.0, + "36": 1226185728.0, + "37": 1226185728.0, + "38": 1226185728.0, + "39": 1226185728.0, + "40": 1226185728.0, + "41": 1226185728.0, + "42": 1226185728.0, + "43": 1226185728.0, + "44": 1226185728.0, + "45": 1226185728.0, + "46": 1226185728.0, + "47": 1226185728.0, + "48": 1226185728.0, + "49": 1226185728.0, + "50": 1226185728.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.92495, - "3": 0.42609, - "4": 0.38164, - "5": 0.38243, - "6": 0.39245, - "7": 0.37703, - "8": 0.37227, - "9": 0.37806, - "10": 0.37329, - "11": 0.37327, - "12": 0.37137, - "13": 0.37408, - "14": 0.37041, - "15": 0.36981, - "16": 0.37016, - "17": 0.37042, - "18": 0.36831, - "19": 0.371, - "20": 0.37313, - "21": 0.36719, - "22": 0.36868, - "23": 0.36814, - "24": 0.37364, - "25": 0.40111, - "26": 0.38183, - "27": 0.37268, - "28": 0.36961, - "29": 0.37033, - "30": 0.37065, - "31": 0.37349, - "32": 0.37083, - "33": 0.37375, - "34": 0.37117, - "35": 0.37042, - "36": 0.37079, - "37": 0.37509, - "38": 0.36943, - "39": 0.37539, - "40": 0.37163, - "41": 0.3686, - "42": 0.37164, - "43": 0.37063, - "44": 0.37145, - "45": 0.37059, - "46": 0.3672, - "47": 0.37787, - "48": 0.36931, - "49": 0.36865, - "50": 0.36944 + "2": 7.49023, + "3": 0.44027, + "4": 0.41145, + "5": 0.41698, + "6": 0.41714, + "7": 0.40413, + "8": 0.40242, + "9": 0.40546, + "10": 0.40167, + "11": 0.40186, + "12": 0.40015, + "13": 0.40174, + "14": 0.40713, + "15": 0.40037, + "16": 0.39929, + "17": 0.39991, + "18": 0.40206, + "19": 0.39647, + "20": 0.40679, + "21": 0.40028, + "22": 0.39725, + "23": 0.48653, + "24": 0.43489, + "25": 0.39323, + "26": 0.39394, + "27": 0.40531, + "28": 0.39851, + "29": 0.44081, + "30": 0.45963, + "31": 0.41177, + "32": 0.39697, + "33": 0.4811, + "34": 0.40673, + "35": 0.39929, + "36": 0.40487, + "37": 0.40857, + "38": 0.40206, + "39": 0.40031, + "40": 0.39543, + "41": 0.3944, + "42": 0.39567, + "43": 0.39744, + "44": 0.3997, + "45": 0.39518, + "46": 0.394, + "47": 0.40243, + "48": 0.39645, + "49": 0.39612, + "50": 0.39612 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json index d12d93c5868..579870d9a79 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_a100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.9191, - "2": 10.91657, - "3": 10.92612, - "4": 10.91743, - "5": 10.92515, - "6": 10.91455, - "7": 10.91757, - "8": 10.9188, - "9": 10.9198, - "10": 10.92142, - "11": 10.91544, - "12": 10.91366, - "13": 10.89765, - "14": 10.90237, - "15": 10.8841, - "16": 10.88468, - "17": 10.87811, - "18": 10.87305, - "19": 10.86617, - "20": 10.79818, - "21": 10.79896, - "22": 10.78314, - "23": 10.78596, - "24": 10.74431, - "25": 10.76005, - "26": 10.72959, - "27": 10.7235, - "28": 10.64773, - "29": 10.61837, - "30": 10.59978, - "31": 10.60059, - "32": 10.57623, - "33": 10.55868, - "34": 10.51277, - "35": 10.51753, - "36": 10.51378, - "37": 10.47335, - "38": 10.46866, - "39": 10.45387, - "40": 10.43424, - "41": 10.4083, - "42": 10.39229, - "43": 10.37761, - "44": 10.34934, - "45": 10.35892, - "46": 10.31862, - "47": 10.30601, - "48": 10.26806, - "49": 10.25481, - "50": 10.26589 + "1": 10.92007, + "2": 10.91646, + "3": 10.92617, + "4": 10.91763, + "5": 10.92559, + "6": 10.91443, + "7": 10.91661, + "8": 10.91832, + "9": 10.92012, + "10": 10.92144, + "11": 10.91542, + "12": 10.91428, + "13": 10.89766, + "14": 10.90217, + "15": 10.88415, + "16": 10.88446, + "17": 10.87736, + "18": 10.87282, + "19": 10.86559, + "20": 10.79824, + "21": 10.79883, + "22": 10.7828, + "23": 10.7852, + "24": 10.74413, + "25": 10.75902, + "26": 10.7296, + "27": 10.72396, + "28": 10.64747, + "29": 10.61781, + "30": 10.59955, + "31": 10.60032, + "32": 10.57668, + "33": 10.55896, + "34": 10.51264, + "35": 10.51768, + "36": 10.51356, + "37": 10.47325, + "38": 10.46901, + "39": 10.45416, + "40": 10.43408, + "41": 10.40788, + "42": 10.39227, + "43": 10.37741, + "44": 10.34965, + "45": 10.35859, + "46": 10.31887, + "47": 10.30644, + "48": 10.26835, + "49": 10.25488, + "50": 10.26582 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6636.0, - "2": 6475.0, - "3": 6570.0, - "4": 6636.0, - "5": 6399.0, - "6": 6400.0, - "7": 6829.0, - "8": 6547.0, - "9": 6746.0, - "10": 6769.0, - "11": 6595.0, - "12": 6215.0, - "13": 6488.0, - "14": 6605.0, - "15": 6555.0, - "16": 6724.0, - "17": 6683.0, - "18": 6720.0, - "19": 6686.0, - "20": 6446.0, - "21": 6643.0, - "22": 6786.0, - "23": 6769.0, - "24": 6544.0, - "25": 6583.0, - "26": 6630.0, - "27": 6774.0, - "28": 6957.0, - "29": 6987.0, - "30": 6549.0, - "31": 7066.0, - "32": 6858.0, - "33": 6666.0, - "34": 6949.0, - "35": 7007.0, - "36": 6782.0, - "37": 7318.0, - "38": 7191.0, - "39": 7055.0, - "40": 7486.0, - "41": 7440.0, - "42": 6901.0, - "43": 7459.0, - "44": 7117.0, - "45": 8044.0, - "46": 7520.0, - "47": 7744.0, - "48": 8052.0, - "49": 8201.0, - "50": 7942.0 + "1": 6551.0, + "2": 6568.0, + "3": 6658.0, + "4": 6484.0, + "5": 6416.0, + "6": 6449.0, + "7": 6849.0, + "8": 6523.0, + "9": 6714.0, + "10": 6721.0, + "11": 6740.0, + "12": 6438.0, + "13": 6835.0, + "14": 6580.0, + "15": 6483.0, + "16": 6662.0, + "17": 6613.0, + "18": 6616.0, + "19": 6794.0, + "20": 6476.0, + "21": 6779.0, + "22": 6751.0, + "23": 6651.0, + "24": 6595.0, + "25": 6516.0, + "26": 6755.0, + "27": 6669.0, + "28": 6867.0, + "29": 6905.0, + "30": 6609.0, + "31": 7012.0, + "32": 6827.0, + "33": 6818.0, + "34": 6950.0, + "35": 6838.0, + "36": 6743.0, + "37": 7117.0, + "38": 7026.0, + "39": 7200.0, + "40": 7335.0, + "41": 7388.0, + "42": 6842.0, + "43": 7461.0, + "44": 7177.0, + "45": 7972.0, + "46": 7532.0, + "47": 7653.0, + "48": 7939.0, + "49": 8325.0, + "50": 7914.0 } }, "mem-allocated-bytes": { @@ -119,53 +119,53 @@ "step_interval": 1, "values": { "1": 462408192.0, - "2": 462408704.0, - "3": 462409216.0, - "4": 462406144.0, + "2": 462409728.0, + "3": 462408704.0, + "4": 462407680.0, "5": 462408192.0, "6": 462406656.0, - "7": 462409216.0, + "7": 462410240.0, "8": 462409216.0, "9": 462406656.0, - "10": 462407168.0, - "11": 462408704.0, - "12": 462408192.0, - "13": 462409728.0, + "10": 462406656.0, + "11": 462408192.0, + "12": 462407168.0, + "13": 462409216.0, "14": 462409728.0, "15": 462408192.0, - "16": 462408192.0, - "17": 462408192.0, + "16": 462407168.0, + "17": 462409216.0, "18": 462409728.0, "19": 462409216.0, "20": 462409216.0, "21": 462406656.0, "22": 462410240.0, "23": 462408192.0, - "24": 462407680.0, - "25": 462408704.0, + "24": 462406656.0, + "25": 462409728.0, "26": 462406656.0, - "27": 462407680.0, + "27": 462408192.0, "28": 462408192.0, - "29": 462408704.0, - "30": 462408192.0, + "29": 462408192.0, + "30": 462409216.0, "31": 462409216.0, "32": 462408704.0, "33": 462407168.0, "34": 462408192.0, - "35": 462407168.0, + "35": 462406656.0, "36": 462407680.0, "37": 462406656.0, - "38": 462408704.0, - "39": 462408192.0, + "38": 462408192.0, + "39": 462409216.0, "40": 462409216.0, "41": 462408704.0, - "42": 462408192.0, - "43": 462407680.0, - "44": 462406656.0, - "45": 462409216.0, - "46": 462408192.0, - "47": 462408192.0, - "48": 462408704.0, + "42": 462407168.0, + "43": 462408704.0, + "44": 462407680.0, + "45": 462410240.0, + "46": 462408704.0, + "47": 462407680.0, + "48": 462407680.0, "49": 462409728.0, "50": 462409216.0 } @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1032205824.0, - "2": 1196511744.0, - "3": 1196511744.0, - "4": 1196529664.0, - "5": 1196529664.0, - "6": 1196529664.0, - "7": 1196529664.0, - "8": 1196529664.0, - "9": 1196605440.0, - "10": 1196605440.0, - "11": 1196605440.0, - "12": 1196605440.0, - "13": 1196605440.0, - "14": 1196605440.0, - "15": 1196605440.0, - "16": 1196605440.0, - "17": 1196605440.0, - "18": 1196605440.0, - "19": 1196605440.0, - "20": 1196605440.0, - "21": 1196605440.0, - "22": 1196605440.0, - "23": 1196605440.0, - "24": 1196605440.0, - "25": 1196605440.0, - "26": 1196605440.0, - "27": 1196605440.0, - "28": 1196605440.0, - "29": 1196605440.0, - "30": 1196605440.0, - "31": 1196605440.0, - "32": 1196605440.0, - "33": 1197258240.0, - "34": 1197258240.0, - "35": 1197258240.0, - "36": 1197258240.0, - "37": 1197258240.0, - "38": 1197258240.0, - "39": 1197258240.0, - "40": 1197258240.0, - "41": 1197258240.0, - "42": 1197258240.0, - "43": 1197258240.0, - "44": 1197258240.0, - "45": 1197258240.0, - "46": 1197258240.0, - "47": 1197258240.0, - "48": 1197258240.0, - "49": 1197258240.0, - "50": 1197258240.0 + "1": 1032228352.0, + "2": 1196615168.0, + "3": 1196615168.0, + "4": 1196615168.0, + "5": 1196615168.0, + "6": 1196615168.0, + "7": 1196615168.0, + "8": 1196615168.0, + "9": 1196615168.0, + "10": 1196630528.0, + "11": 1196630528.0, + "12": 1196630528.0, + "13": 1196630528.0, + "14": 1196630528.0, + "15": 1196630528.0, + "16": 1196630528.0, + "17": 1196630528.0, + "18": 1196630528.0, + "19": 1196630528.0, + "20": 1196630528.0, + "21": 1196630528.0, + "22": 1196630528.0, + "23": 1196630528.0, + "24": 1196630528.0, + "25": 1196630528.0, + "26": 1196630528.0, + "27": 1196630528.0, + "28": 1196630528.0, + "29": 1196630528.0, + "30": 1196630528.0, + "31": 1196630528.0, + "32": 1196630528.0, + "33": 1197310464.0, + "34": 1197310464.0, + "35": 1197310464.0, + "36": 1197310464.0, + "37": 1197310464.0, + "38": 1197310464.0, + "39": 1197310464.0, + "40": 1197310464.0, + "41": 1197310464.0, + "42": 1197310464.0, + "43": 1197310464.0, + "44": 1197310464.0, + "45": 1197310464.0, + "46": 1197310464.0, + "47": 1197310464.0, + "48": 1197310464.0, + "49": 1197310464.0, + "50": 1197310464.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.53559, - "3": 0.56564, - "4": 0.56573, - "5": 0.55332, - "6": 0.55503, - "7": 0.55133, - "8": 0.5489, - "9": 0.55114, - "10": 0.54846, - "11": 0.54866, - "12": 0.54873, - "13": 0.54959, - "14": 0.54824, - "15": 0.55007, - "16": 0.54967, - "17": 0.54824, - "18": 0.54844, - "19": 0.54849, - "20": 0.54863, - "21": 0.55145, - "22": 0.54909, - "23": 0.55, - "24": 0.551, - "25": 0.54859, - "26": 0.54959, - "27": 0.55167, - "28": 0.54886, - "29": 0.54826, - "30": 0.54978, - "31": 0.55026, - "32": 0.5508, - "33": 0.55066, - "34": 0.55058, - "35": 0.55095, - "36": 0.54936, - "37": 0.54911, - "38": 0.5486, - "39": 0.54872, - "40": 0.54899, - "41": 0.54852, - "42": 0.54752, - "43": 0.54831, - "44": 0.54783, - "45": 0.54743, - "46": 0.54757, - "47": 0.54742, - "48": 0.54759, - "49": 0.54903, - "50": 0.54559 + "2": 6.70856, + "3": 0.58853, + "4": 0.56801, + "5": 0.56862, + "6": 0.57006, + "7": 0.56937, + "8": 0.56871, + "9": 0.56699, + "10": 0.56762, + "11": 0.56686, + "12": 0.57463, + "13": 0.56889, + "14": 0.56857, + "15": 0.56734, + "16": 0.56822, + "17": 0.56633, + "18": 0.56816, + "19": 0.56851, + "20": 0.56951, + "21": 0.56686, + "22": 0.56905, + "23": 0.56783, + "24": 0.56651, + "25": 0.56889, + "26": 0.5673, + "27": 0.56746, + "28": 0.56838, + "29": 0.56653, + "30": 0.56734, + "31": 0.56831, + "32": 0.56842, + "33": 0.56942, + "34": 0.56804, + "35": 0.56787, + "36": 0.56781, + "37": 0.56774, + "38": 0.56616, + "39": 0.56898, + "40": 0.57278, + "41": 0.58315, + "42": 0.57621, + "43": 0.56839, + "44": 0.5703, + "45": 0.56846, + "46": 0.56701, + "47": 0.56954, + "48": 0.57414, + "49": 0.57061, + "50": 0.56972 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json index 9fb4464c4f6..1f57f5e1132 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92051, - "2": 10.92395, - "3": 10.92254, - "4": 10.92049, - "5": 10.91474, - "6": 10.90369, - "7": 10.9112, - "8": 10.91582, - "9": 10.91966, - "10": 10.91909, - "11": 10.90563, - "12": 10.90649, - "13": 10.90489, - "14": 10.89897, - "15": 10.87714, - "16": 10.87001, - "17": 10.8834, - "18": 10.86628, - "19": 10.86686, + "1": 10.92142, + "2": 10.92363, + "3": 10.92242, + "4": 10.92026, + "5": 10.91483, + "6": 10.90385, + "7": 10.91109, + "8": 10.91595, + "9": 10.92047, + "10": 10.91896, + "11": 10.90617, + "12": 10.90579, + "13": 10.90562, + "14": 10.8989, + "15": 10.87723, + "16": 10.86957, + "17": 10.8846, + "18": 10.86603, + "19": 10.86663, "20": 10.80107, "21": 10.79745, - "22": 10.78577, - "23": 10.77879, - "24": 10.74333, - "25": 10.74705, - "26": 10.7275, - "27": 10.70964, - "28": 10.64399, - "29": 10.60607, - "30": 10.59747, - "31": 10.58617, - "32": 10.57896, - "33": 10.54631, - "34": 10.50565, - "35": 10.50708, - "36": 10.49921, - "37": 10.45846, - "38": 10.4669, - "39": 10.43069, - "40": 10.42274, - "41": 10.40014, - "42": 10.38466, - "43": 10.36939, - "44": 10.34001, - "45": 10.34882, - "46": 10.31053, - "47": 10.29582, - "48": 10.2503, - "49": 10.24405, - "50": 10.25168 + "22": 10.78608, + "23": 10.77958, + "24": 10.74282, + "25": 10.74782, + "26": 10.72794, + "27": 10.70948, + "28": 10.64378, + "29": 10.60676, + "30": 10.59805, + "31": 10.58664, + "32": 10.57936, + "33": 10.546, + "34": 10.50679, + "35": 10.50654, + "36": 10.49965, + "37": 10.45837, + "38": 10.46672, + "39": 10.43117, + "40": 10.42314, + "41": 10.4007, + "42": 10.38462, + "43": 10.36964, + "44": 10.34028, + "45": 10.34887, + "46": 10.31013, + "47": 10.29593, + "48": 10.25007, + "49": 10.24404, + "50": 10.25153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6472.0, - "2": 6590.0, - "3": 6715.0, - "4": 6477.0, - "5": 6647.0, - "6": 6417.0, - "7": 6895.0, - "8": 6525.0, - "9": 6635.0, - "10": 6725.0, - "11": 6782.0, - "12": 6345.0, - "13": 6855.0, - "14": 6764.0, - "15": 6484.0, - "16": 6478.0, - "17": 6699.0, - "18": 6486.0, - "19": 6797.0, - "20": 6592.0, - "21": 6504.0, - "22": 6673.0, - "23": 6620.0, - "24": 6525.0, - "25": 6471.0, - "26": 6508.0, - "27": 6578.0, - "28": 6593.0, - "29": 6541.0, - "30": 6573.0, - "31": 6920.0, - "32": 6681.0, - "33": 6825.0, - "34": 7093.0, - "35": 7100.0, - "36": 6669.0, - "37": 7293.0, - "38": 7094.0, - "39": 7224.0, - "40": 7424.0, - "41": 7389.0, - "42": 6727.0, - "43": 7386.0, - "44": 7247.0, - "45": 7896.0, - "46": 7544.0, - "47": 7638.0, - "48": 7890.0, - "49": 8076.0, - "50": 7894.0 + "1": 6494.0, + "2": 6740.0, + "3": 6956.0, + "4": 6747.0, + "5": 6648.0, + "6": 6407.0, + "7": 6764.0, + "8": 6471.0, + "9": 6733.0, + "10": 6750.0, + "11": 6865.0, + "12": 6288.0, + "13": 6782.0, + "14": 6749.0, + "15": 6437.0, + "16": 6530.0, + "17": 6644.0, + "18": 6336.0, + "19": 6734.0, + "20": 6356.0, + "21": 6588.0, + "22": 6652.0, + "23": 6530.0, + "24": 6665.0, + "25": 6702.0, + "26": 6687.0, + "27": 6681.0, + "28": 6510.0, + "29": 6670.0, + "30": 6556.0, + "31": 6782.0, + "32": 6863.0, + "33": 6884.0, + "34": 7126.0, + "35": 7032.0, + "36": 6895.0, + "37": 7186.0, + "38": 6999.0, + "39": 7246.0, + "40": 7364.0, + "41": 7488.0, + "42": 6879.0, + "43": 7620.0, + "44": 7231.0, + "45": 7956.0, + "46": 7554.0, + "47": 7513.0, + "48": 7742.0, + "49": 8157.0, + "50": 7799.0 } }, "mem-allocated-bytes": { @@ -119,55 +119,55 @@ "step_interval": 1, "values": { "1": 491766272.0, - "2": 491768320.0, + "2": 491769344.0, "3": 491768320.0, "4": 491769856.0, - "5": 491770368.0, - "6": 491766784.0, - "7": 491766272.0, - "8": 491766272.0, + "5": 491768320.0, + "6": 491767808.0, + "7": 491765248.0, + "8": 491767296.0, "9": 491768832.0, "10": 491769344.0, "11": 491768832.0, - "12": 491768832.0, - "13": 491767296.0, + "12": 491769856.0, + "13": 491766784.0, "14": 491769344.0, "15": 491769344.0, "16": 491767808.0, "17": 491767296.0, - "18": 491768832.0, + "18": 491769344.0, "19": 491769344.0, - "20": 491769344.0, + "20": 491768832.0, "21": 491765760.0, "22": 491770880.0, "23": 491767808.0, - "24": 491768320.0, + "24": 491768832.0, "25": 491769344.0, - "26": 491767296.0, + "26": 491766784.0, "27": 491768320.0, "28": 491767808.0, "29": 491766272.0, - "30": 491769344.0, - "31": 491770880.0, - "32": 491768320.0, - "33": 491767808.0, - "34": 491766784.0, - "35": 491768320.0, + "30": 491768832.0, + "31": 491768832.0, + "32": 491769856.0, + "33": 491768320.0, + "34": 491768832.0, + "35": 491769344.0, "36": 491770368.0, - "37": 491769344.0, + "37": 491768320.0, "38": 491769856.0, "39": 491767808.0, - "40": 491767808.0, - "41": 491770880.0, - "42": 491771392.0, - "43": 491768832.0, + "40": 491767296.0, + "41": 491769856.0, + "42": 491770368.0, + "43": 491767808.0, "44": 491768832.0, - "45": 491769344.0, + "45": 491770368.0, "46": 491771392.0, "47": 491769344.0, "48": 491770368.0, "49": 491770368.0, - "50": 491769856.0 + "50": 491768832.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1059454976.0, - "2": 1225387520.0, - "3": 1225387520.0, - "4": 1225387520.0, - "5": 1225387520.0, - "6": 1226523136.0, - "7": 1226523136.0, - "8": 1226523136.0, - "9": 1226523136.0, - "10": 1226523136.0, - "11": 1226523136.0, - "12": 1226523136.0, - "13": 1226523136.0, - "14": 1226523136.0, - "15": 1226523136.0, - "16": 1226523136.0, - "17": 1226523136.0, - "18": 1226523136.0, - "19": 1226523136.0, - "20": 1226523136.0, - "21": 1226523136.0, - "22": 1226523136.0, - "23": 1226523136.0, - "24": 1226523136.0, - "25": 1226523136.0, - "26": 1226523136.0, - "27": 1226523136.0, - "28": 1226523136.0, - "29": 1226523136.0, - "30": 1226523136.0, - "31": 1226523136.0, - "32": 1226523136.0, - "33": 1226523136.0, - "34": 1226523136.0, - "35": 1226523136.0, - "36": 1226523136.0, - "37": 1226523136.0, - "38": 1226523136.0, - "39": 1226523136.0, - "40": 1226523136.0, - "41": 1226523136.0, - "42": 1226523136.0, - "43": 1226523136.0, - "44": 1226523136.0, - "45": 1226523136.0, - "46": 1226523136.0, - "47": 1226523136.0, - "48": 1226523136.0, - "49": 1226523136.0, - "50": 1226523136.0 + "1": 1060007936.0, + "2": 1225918976.0, + "3": 1225918976.0, + "4": 1226195456.0, + "5": 1226195456.0, + "6": 1226195456.0, + "7": 1226195456.0, + "8": 1226195456.0, + "9": 1226195456.0, + "10": 1226195456.0, + "11": 1226195456.0, + "12": 1226195456.0, + "13": 1226195456.0, + "14": 1226288128.0, + "15": 1226288128.0, + "16": 1226288128.0, + "17": 1226288128.0, + "18": 1226288128.0, + "19": 1226288128.0, + "20": 1226370048.0, + "21": 1226370048.0, + "22": 1226370048.0, + "23": 1226370048.0, + "24": 1226370048.0, + "25": 1226370048.0, + "26": 1226370048.0, + "27": 1226370048.0, + "28": 1226370048.0, + "29": 1226370048.0, + "30": 1226370048.0, + "31": 1226370048.0, + "32": 1226370048.0, + "33": 1226413056.0, + "34": 1226413056.0, + "35": 1226587136.0, + "36": 1226587136.0, + "37": 1226587136.0, + "38": 1226587136.0, + "39": 1226587136.0, + "40": 1226587136.0, + "41": 1226592768.0, + "42": 1226592768.0, + "43": 1226592768.0, + "44": 1226592768.0, + "45": 1226592768.0, + "46": 1226592768.0, + "47": 1226592768.0, + "48": 1226592768.0, + "49": 1226592768.0, + "50": 1226592768.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.52651, - "3": 0.70797, - "4": 0.66493, - "5": 0.657, - "6": 0.65629, - "7": 0.65266, - "8": 0.66048, - "9": 0.65354, - "10": 0.6539, - "11": 0.81197, - "12": 0.89522, - "13": 0.66263, - "14": 0.65332, - "15": 0.65403, - "16": 0.65643, - "17": 0.65374, - "18": 0.6549, - "19": 0.65353, - "20": 0.65446, - "21": 0.65566, - "22": 0.6555, - "23": 0.65133, - "24": 0.65367, - "25": 0.65046, - "26": 0.65286, - "27": 0.65513, - "28": 0.65421, - "29": 0.65491, - "30": 0.65579, - "31": 0.65334, - "32": 0.65505, - "33": 0.65603, - "34": 0.65821, - "35": 0.6583, - "36": 0.65496, - "37": 0.65418, - "38": 0.65686, - "39": 0.65402, - "40": 0.66576, - "41": 0.65538, - "42": 0.65321, - "43": 0.65623, - "44": 0.65842, - "45": 0.65817, - "46": 0.66281, - "47": 0.65702, - "48": 0.66118, - "49": 0.65325, - "50": 0.65619 + "2": 11.60541, + "3": 0.61291, + "4": 0.5572, + "5": 0.55396, + "6": 0.55254, + "7": 0.54829, + "8": 0.56238, + "9": 0.54714, + "10": 0.54831, + "11": 0.54217, + "12": 0.54851, + "13": 0.55042, + "14": 0.5469, + "15": 0.54587, + "16": 0.53903, + "17": 0.54316, + "18": 0.54348, + "19": 0.54321, + "20": 0.54422, + "21": 0.54246, + "22": 0.5412, + "23": 0.55211, + "24": 0.54471, + "25": 0.54094, + "26": 0.54146, + "27": 0.53959, + "28": 0.5428, + "29": 0.54256, + "30": 0.55159, + "31": 0.54673, + "32": 0.54362, + "33": 0.54226, + "34": 0.53984, + "35": 0.54136, + "36": 0.54209, + "37": 0.54121, + "38": 0.54212, + "39": 0.54571, + "40": 0.53737, + "41": 0.54334, + "42": 0.54371, + "43": 0.54174, + "44": 0.54689, + "45": 0.54505, + "46": 0.54406, + "47": 0.54413, + "48": 0.54286, + "49": 0.54678, + "50": 0.54026 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json index f5f37f0f8a9..5ca1b345f29 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94605, - "2": 10.93984, - "3": 10.94844, - "4": 10.94383, - "5": 10.94536, - "6": 10.94551, - "7": 10.94076, - "8": 10.93525, - "9": 10.93574, - "10": 10.93995, - "11": 10.9232, - "12": 10.92749, - "13": 10.93106, - "14": 10.92607, - "15": 10.90323, - "16": 10.88838, - "17": 10.90185, - "18": 10.89577, - "19": 10.89614, - "20": 10.80805, - "21": 10.8201, - "22": 10.80485, - "23": 10.80528, - "24": 10.76201, - "25": 10.77306, - "26": 10.74874, - "27": 10.73469, - "28": 10.67415, - "29": 10.64362, - "30": 10.6161, - "31": 10.60746, - "32": 10.59793, - "33": 10.56729, - "34": 10.53273, - "35": 10.53545, - "36": 10.52116, - "37": 10.48954, - "38": 10.48719, - "39": 10.46139, - "40": 10.44216, - "41": 10.4234, - "42": 10.41064, - "43": 10.39619, - "44": 10.36242, - "45": 10.37314, - "46": 10.33605, - "47": 10.32291, - "48": 10.28505, - "49": 10.27029, - "50": 10.27674 + "1": 10.94588, + "2": 10.93966, + "3": 10.94811, + "4": 10.94371, + "5": 10.94538, + "6": 10.94555, + "7": 10.94068, + "8": 10.93521, + "9": 10.93557, + "10": 10.93962, + "11": 10.92315, + "12": 10.92793, + "13": 10.93059, + "14": 10.92558, + "15": 10.90423, + "16": 10.88742, + "17": 10.90114, + "18": 10.89548, + "19": 10.89627, + "20": 10.80893, + "21": 10.82025, + "22": 10.80528, + "23": 10.80526, + "24": 10.76273, + "25": 10.77254, + "26": 10.74899, + "27": 10.73484, + "28": 10.67376, + "29": 10.64442, + "30": 10.61672, + "31": 10.60725, + "32": 10.59762, + "33": 10.56673, + "34": 10.53272, + "35": 10.53538, + "36": 10.52082, + "37": 10.48908, + "38": 10.48669, + "39": 10.46215, + "40": 10.44264, + "41": 10.42395, + "42": 10.4105, + "43": 10.3965, + "44": 10.36255, + "45": 10.373, + "46": 10.33641, + "47": 10.32243, + "48": 10.28518, + "49": 10.27081, + "50": 10.27664 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6735.0, - "2": 6805.0, - "3": 6618.0, - "4": 6774.0, - "5": 6712.0, - "6": 6531.0, - "7": 6946.0, - "8": 6655.0, - "9": 6914.0, - "10": 6647.0, - "11": 6828.0, - "12": 6387.0, - "13": 6821.0, - "14": 6724.0, - "15": 6698.0, - "16": 6520.0, - "17": 6900.0, - "18": 6532.0, - "19": 6843.0, - "20": 6411.0, - "21": 6559.0, - "22": 6525.0, - "23": 6655.0, - "24": 6550.0, - "25": 6633.0, - "26": 6637.0, - "27": 6765.0, - "28": 6593.0, - "29": 6801.0, - "30": 6515.0, - "31": 6821.0, - "32": 6859.0, - "33": 6841.0, - "34": 6861.0, - "35": 7014.0, - "36": 6780.0, - "37": 7226.0, - "38": 7060.0, - "39": 7169.0, - "40": 7319.0, - "41": 7417.0, - "42": 6748.0, - "43": 7596.0, - "44": 7102.0, - "45": 7827.0, - "46": 7407.0, - "47": 7564.0, - "48": 7726.0, - "49": 8052.0, - "50": 7755.0 + "1": 6781.0, + "2": 6861.0, + "3": 6659.0, + "4": 6776.0, + "5": 6623.0, + "6": 6603.0, + "7": 6865.0, + "8": 6613.0, + "9": 6778.0, + "10": 6831.0, + "11": 6880.0, + "12": 6623.0, + "13": 6913.0, + "14": 6919.0, + "15": 6457.0, + "16": 6555.0, + "17": 6782.0, + "18": 6633.0, + "19": 6858.0, + "20": 6320.0, + "21": 6561.0, + "22": 6510.0, + "23": 6652.0, + "24": 6702.0, + "25": 6433.0, + "26": 6613.0, + "27": 6579.0, + "28": 6640.0, + "29": 6909.0, + "30": 6622.0, + "31": 6744.0, + "32": 6820.0, + "33": 6721.0, + "34": 6972.0, + "35": 6901.0, + "36": 6868.0, + "37": 7481.0, + "38": 6965.0, + "39": 7181.0, + "40": 7424.0, + "41": 7386.0, + "42": 6905.0, + "43": 7306.0, + "44": 6939.0, + "45": 7642.0, + "46": 7417.0, + "47": 7607.0, + "48": 7685.0, + "49": 8041.0, + "50": 7703.0 } }, "mem-allocated-bytes": { @@ -118,53 +118,53 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 491769344.0, - "2": 491769344.0, - "3": 491768320.0, - "4": 491768832.0, + "1": 491769856.0, + "2": 491769856.0, + "3": 491768832.0, + "4": 491767296.0, "5": 491768832.0, "6": 491769856.0, - "7": 491767808.0, - "8": 491768320.0, - "9": 491767296.0, + "7": 491766784.0, + "8": 491767296.0, + "9": 491768320.0, "10": 491767296.0, - "11": 491767808.0, + "11": 491766784.0, "12": 491768320.0, - "13": 491767296.0, + "13": 491769344.0, "14": 491766784.0, - "15": 491769856.0, - "16": 491769344.0, + "15": 491770880.0, + "16": 491768320.0, "17": 491767808.0, "18": 491768320.0, "19": 491767296.0, "20": 491766784.0, - "21": 491767296.0, - "22": 491768320.0, + "21": 491768832.0, + "22": 491768832.0, "23": 491768320.0, "24": 491767296.0, "25": 491766272.0, - "26": 491767296.0, - "27": 491767808.0, + "26": 491765760.0, + "27": 491766784.0, "28": 491768832.0, "29": 491769344.0, - "30": 491768320.0, - "31": 491767296.0, - "32": 491769344.0, - "33": 491768320.0, - "34": 491766784.0, + "30": 491769344.0, + "31": 491766272.0, + "32": 491768832.0, + "33": 491767808.0, + "34": 491767808.0, "35": 491766784.0, - "36": 491766272.0, + "36": 491766784.0, "37": 491767808.0, - "38": 491768832.0, + "38": 491768320.0, "39": 491766784.0, "40": 491766784.0, "41": 491767808.0, "42": 491766784.0, "43": 491765760.0, "44": 491768320.0, - "45": 491768320.0, - "46": 491768832.0, - "47": 491768320.0, + "45": 491769856.0, + "46": 491769856.0, + "47": 491768832.0, "48": 491766784.0, "49": 491769856.0, "50": 491767296.0 @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1059374080.0, - "2": 1225970688.0, - "3": 1225970688.0, - "4": 1225970688.0, - "5": 1225970688.0, - "6": 1225970688.0, - "7": 1225970688.0, - "8": 1225970688.0, - "9": 1225970688.0, - "10": 1225970688.0, - "11": 1225970688.0, - "12": 1225970688.0, - "13": 1225970688.0, - "14": 1225970688.0, - "15": 1225970688.0, - "16": 1225970688.0, - "17": 1225970688.0, - "18": 1225970688.0, - "19": 1225970688.0, - "20": 1225970688.0, - "21": 1225970688.0, - "22": 1225970688.0, - "23": 1225970688.0, - "24": 1225970688.0, - "25": 1225970688.0, - "26": 1225970688.0, - "27": 1225970688.0, - "28": 1225970688.0, - "29": 1225970688.0, - "30": 1225970688.0, - "31": 1225970688.0, - "32": 1225970688.0, - "33": 1225970688.0, - "34": 1225970688.0, - "35": 1225970688.0, - "36": 1225970688.0, - "37": 1225970688.0, - "38": 1225970688.0, - "39": 1225970688.0, - "40": 1225970688.0, - "41": 1225970688.0, - "42": 1225970688.0, - "43": 1225970688.0, - "44": 1225970688.0, - "45": 1225970688.0, - "46": 1225970688.0, - "47": 1225970688.0, - "48": 1225970688.0, - "49": 1225970688.0, - "50": 1225970688.0 + "1": 1060344832.0, + "2": 1225047040.0, + "3": 1225791488.0, + "4": 1225791488.0, + "5": 1225791488.0, + "6": 1225791488.0, + "7": 1225791488.0, + "8": 1225791488.0, + "9": 1225831936.0, + "10": 1225831936.0, + "11": 1225831936.0, + "12": 1225831936.0, + "13": 1225831936.0, + "14": 1225831936.0, + "15": 1225831936.0, + "16": 1225831936.0, + "17": 1225831936.0, + "18": 1225831936.0, + "19": 1225831936.0, + "20": 1225831936.0, + "21": 1225831936.0, + "22": 1225831936.0, + "23": 1225831936.0, + "24": 1225831936.0, + "25": 1225831936.0, + "26": 1225831936.0, + "27": 1226185728.0, + "28": 1226185728.0, + "29": 1226185728.0, + "30": 1226185728.0, + "31": 1226185728.0, + "32": 1226185728.0, + "33": 1226185728.0, + "34": 1226185728.0, + "35": 1226185728.0, + "36": 1226185728.0, + "37": 1226185728.0, + "38": 1226185728.0, + "39": 1226185728.0, + "40": 1226185728.0, + "41": 1226185728.0, + "42": 1226185728.0, + "43": 1226185728.0, + "44": 1226185728.0, + "45": 1226185728.0, + "46": 1226185728.0, + "47": 1226185728.0, + "48": 1226185728.0, + "49": 1226185728.0, + "50": 1226185728.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.69842, - "3": 0.42955, - "4": 0.38254, - "5": 0.38294, - "6": 0.40484, - "7": 0.378, - "8": 0.37223, - "9": 0.38117, - "10": 0.37296, - "11": 0.37588, - "12": 0.37449, - "13": 0.36975, - "14": 0.37177, - "15": 0.36974, - "16": 0.37045, - "17": 0.37094, - "18": 0.36964, - "19": 0.3717, - "20": 0.3753, - "21": 0.37275, - "22": 0.369, - "23": 0.36994, - "24": 0.3728, - "25": 0.37072, - "26": 0.37731, - "27": 0.37465, - "28": 0.37369, - "29": 0.36972, - "30": 0.37074, - "31": 0.37919, - "32": 0.37175, - "33": 0.37389, - "34": 0.36981, - "35": 0.37196, - "36": 0.37185, - "37": 0.37467, - "38": 0.37303, - "39": 0.37903, - "40": 0.37475, - "41": 0.37008, - "42": 0.37249, - "43": 0.36924, - "44": 0.36802, - "45": 0.37311, - "46": 0.36911, - "47": 0.37807, - "48": 0.37532, - "49": 0.37321, - "50": 0.37073 + "2": 7.39221, + "3": 0.43771, + "4": 0.40173, + "5": 0.41245, + "6": 0.46335, + "7": 0.40036, + "8": 0.39808, + "9": 0.39527, + "10": 0.40022, + "11": 0.39688, + "12": 0.39609, + "13": 0.39638, + "14": 0.40459, + "15": 0.39274, + "16": 0.39262, + "17": 0.39592, + "18": 0.39398, + "19": 0.39296, + "20": 0.39954, + "21": 0.39581, + "22": 0.39258, + "23": 0.39364, + "24": 0.39431, + "25": 0.39363, + "26": 0.3964, + "27": 0.39292, + "28": 0.3947, + "29": 0.39299, + "30": 0.39396, + "31": 0.39702, + "32": 0.39128, + "33": 0.39383, + "34": 0.39405, + "35": 0.39947, + "36": 0.3924, + "37": 0.39608, + "38": 0.39553, + "39": 0.40084, + "40": 0.39415, + "41": 0.39283, + "42": 0.39479, + "43": 0.39324, + "44": 0.39189, + "45": 0.39304, + "46": 0.39391, + "47": 0.40353, + "48": 0.39764, + "49": 0.39731, + "50": 0.39809 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json index 3df5384b97d..cae80da8dff 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_a100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94389, - "2": 10.92216, - "3": 10.93441, - "4": 10.92997, - "5": 10.94015, - "6": 10.92342, - "7": 10.92917, - "8": 10.92249, - "9": 10.93426, - "10": 10.9266, - "11": 10.92316, - "12": 10.92659, - "13": 10.91374, - "14": 10.91405, - "15": 10.89525, - "16": 10.88753, - "17": 10.88859, - "18": 10.88247, - "19": 10.88009, - "20": 10.81165, - "21": 10.81114, - "22": 10.79045, - "23": 10.7844, - "24": 10.75382, - "25": 10.75295, - "26": 10.74444, - "27": 10.72371, - "28": 10.64913, - "29": 10.62644, - "30": 10.59616, - "31": 10.59853, - "32": 10.5773, - "33": 10.54451, - "34": 10.50919, - "35": 10.51574, - "36": 10.49249, - "37": 10.46371, - "38": 10.45986, - "39": 10.43614, - "40": 10.41221, - "41": 10.39144, - "42": 10.37229, - "43": 10.35418, - "44": 10.33218, - "45": 10.33957, - "46": 10.29487, - "47": 10.28192, - "48": 10.24488, - "49": 10.23641, - "50": 10.23708 + "1": 10.94407, + "2": 10.9223, + "3": 10.93505, + "4": 10.93014, + "5": 10.9399, + "6": 10.9224, + "7": 10.92936, + "8": 10.92318, + "9": 10.93413, + "10": 10.92558, + "11": 10.92238, + "12": 10.92647, + "13": 10.91451, + "14": 10.91494, + "15": 10.89488, + "16": 10.88797, + "17": 10.88818, + "18": 10.88219, + "19": 10.88012, + "20": 10.81183, + "21": 10.81036, + "22": 10.79064, + "23": 10.78433, + "24": 10.75348, + "25": 10.75335, + "26": 10.74461, + "27": 10.724, + "28": 10.64851, + "29": 10.62637, + "30": 10.59604, + "31": 10.59917, + "32": 10.57722, + "33": 10.54471, + "34": 10.50925, + "35": 10.5159, + "36": 10.49251, + "37": 10.46348, + "38": 10.45999, + "39": 10.43553, + "40": 10.41182, + "41": 10.39093, + "42": 10.37203, + "43": 10.35423, + "44": 10.33235, + "45": 10.33963, + "46": 10.29522, + "47": 10.28216, + "48": 10.2451, + "49": 10.23673, + "50": 10.2367 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 15319.0, - "2": 15574.0, - "3": 15533.0, - "4": 15467.0, - "5": 15058.0, - "6": 15382.0, - "7": 16055.0, - "8": 14989.0, - "9": 15337.0, - "10": 15582.0, - "11": 15359.0, - "12": 15103.0, - "13": 16039.0, - "14": 15572.0, - "15": 14935.0, - "16": 14858.0, - "17": 15188.0, - "18": 15433.0, - "19": 15660.0, - "20": 15538.0, - "21": 15372.0, - "22": 15471.0, - "23": 15606.0, - "24": 16044.0, - "25": 15127.0, - "26": 15739.0, - "27": 15666.0, - "28": 15586.0, - "29": 15517.0, - "30": 15894.0, - "31": 16158.0, - "32": 15620.0, - "33": 15615.0, - "34": 15787.0, - "35": 16422.0, - "36": 15989.0, - "37": 16614.0, - "38": 15678.0, - "39": 16629.0, - "40": 16875.0, - "41": 16096.0, - "42": 15976.0, - "43": 17280.0, - "44": 16360.0, - "45": 18959.0, - "46": 17938.0, - "47": 18007.0, - "48": 18948.0, - "49": 20222.0, - "50": 18746.0 + "1": 15209.0, + "2": 15110.0, + "3": 15415.0, + "4": 15561.0, + "5": 15124.0, + "6": 15025.0, + "7": 16272.0, + "8": 14879.0, + "9": 15392.0, + "10": 15551.0, + "11": 15273.0, + "12": 14774.0, + "13": 15460.0, + "14": 15738.0, + "15": 15110.0, + "16": 14974.0, + "17": 15085.0, + "18": 15113.0, + "19": 15816.0, + "20": 15683.0, + "21": 15382.0, + "22": 15364.0, + "23": 15618.0, + "24": 16099.0, + "25": 15184.0, + "26": 15579.0, + "27": 15756.0, + "28": 15289.0, + "29": 15725.0, + "30": 15822.0, + "31": 16275.0, + "32": 15956.0, + "33": 15850.0, + "34": 15872.0, + "35": 16576.0, + "36": 15970.0, + "37": 16336.0, + "38": 15929.0, + "39": 16574.0, + "40": 17280.0, + "41": 16115.0, + "42": 15999.0, + "43": 17327.0, + "44": 16525.0, + "45": 19158.0, + "46": 17903.0, + "47": 17910.0, + "48": 18733.0, + "49": 20183.0, + "50": 18515.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 629057536.0, - "2": 628997120.0, - "3": 628937216.0, - "4": 629266432.0, - "5": 628940288.0, - "6": 629003264.0, - "7": 628958208.0, - "8": 628939776.0, - "9": 628954112.0, - "10": 628938752.0, - "11": 628944896.0, - "12": 628946944.0, - "13": 628940800.0, - "14": 628937728.0, - "15": 628974080.0, - "16": 628987904.0, - "17": 628978688.0, - "18": 629001216.0, - "19": 628939776.0, - "20": 628960256.0, - "21": 628945920.0, - "22": 628938752.0, - "23": 629006848.0, - "24": 628977152.0, - "25": 628969984.0, - "26": 628937728.0, - "27": 628938240.0, - "28": 628964864.0, - "29": 628977664.0, - "30": 628942848.0, - "31": 628980736.0, - "32": 629019136.0, - "33": 629178368.0, - "34": 628936704.0, - "35": 628936192.0, - "36": 628987392.0, - "37": 628994560.0, - "38": 629016576.0, - "39": 628978176.0, - "40": 628937216.0, - "41": 628952576.0, - "42": 628935680.0, - "43": 628936192.0, - "44": 629395968.0, - "45": 628936704.0, - "46": 628975104.0, - "47": 629189120.0, - "48": 628938240.0, - "49": 628949504.0, - "50": 628960768.0 + "1": 629988352.0, + "2": 629986304.0, + "3": 630012416.0, + "4": 630641664.0, + "5": 629988864.0, + "6": 630043648.0, + "7": 630007808.0, + "8": 630068736.0, + "9": 630147584.0, + "10": 629988352.0, + "11": 629995520.0, + "12": 629989888.0, + "13": 630084608.0, + "14": 629992448.0, + "15": 630003712.0, + "16": 630025216.0, + "17": 629999616.0, + "18": 630055936.0, + "19": 630030848.0, + "20": 629984256.0, + "21": 630003712.0, + "22": 629987328.0, + "23": 629994496.0, + "24": 629986816.0, + "25": 630059008.0, + "26": 630021632.0, + "27": 630103040.0, + "28": 629990912.0, + "29": 629989376.0, + "30": 630033920.0, + "31": 630037504.0, + "32": 630054400.0, + "33": 630023168.0, + "34": 630009856.0, + "35": 629984768.0, + "36": 629987328.0, + "37": 629985280.0, + "38": 629986304.0, + "39": 629985792.0, + "40": 630012416.0, + "41": 630017536.0, + "42": 629984256.0, + "43": 629984768.0, + "44": 630265344.0, + "45": 629985792.0, + "46": 630187520.0, + "47": 630078464.0, + "48": 629986816.0, + "49": 629995008.0, + "50": 630007296.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1849006080.0, - "2": 2079139840.0, - "3": 2079139840.0, - "4": 2079139840.0, - "5": 2080343040.0, - "6": 2080343040.0, - "7": 2080343040.0, - "8": 2080343040.0, - "9": 2080343040.0, - "10": 2080343040.0, - "11": 2080343040.0, - "12": 2080343040.0, - "13": 2080343040.0, - "14": 2080343040.0, - "15": 2080343040.0, - "16": 2080343040.0, - "17": 2080343040.0, - "18": 2080343040.0, - "19": 2080343040.0, - "20": 2080343040.0, - "21": 2080343040.0, - "22": 2080343040.0, - "23": 2080343040.0, - "24": 2080343040.0, - "25": 2080343040.0, - "26": 2080343040.0, - "27": 2080343040.0, - "28": 2080343040.0, - "29": 2080343040.0, - "30": 2080343040.0, - "31": 2080343040.0, - "32": 2080418304.0, - "33": 2080418304.0, - "34": 2080418304.0, - "35": 2080418304.0, - "36": 2080418304.0, - "37": 2080418304.0, - "38": 2080418304.0, - "39": 2080418304.0, - "40": 2080418304.0, - "41": 2080418304.0, - "42": 2080418304.0, - "43": 2080418304.0, - "44": 2080418304.0, - "45": 2080418304.0, - "46": 2080418304.0, - "47": 2080418304.0, - "48": 2080418304.0, - "49": 2080418304.0, - "50": 2080418304.0 + "1": 1848761856.0, + "2": 2079795200.0, + "3": 2079795200.0, + "4": 2079795200.0, + "5": 2079956992.0, + "6": 2079956992.0, + "7": 2079956992.0, + "8": 2079956992.0, + "9": 2079956992.0, + "10": 2079956992.0, + "11": 2079956992.0, + "12": 2079956992.0, + "13": 2080411648.0, + "14": 2080411648.0, + "15": 2080411648.0, + "16": 2080411648.0, + "17": 2080728064.0, + "18": 2080728064.0, + "19": 2080728064.0, + "20": 2080728064.0, + "21": 2080728064.0, + "22": 2080728064.0, + "23": 2080728064.0, + "24": 2080728064.0, + "25": 2080728064.0, + "26": 2080728064.0, + "27": 2080728064.0, + "28": 2080728064.0, + "29": 2080728064.0, + "30": 2080728064.0, + "31": 2080728064.0, + "32": 2080728064.0, + "33": 2080728064.0, + "34": 2080728064.0, + "35": 2080728064.0, + "36": 2080728064.0, + "37": 2080728064.0, + "38": 2080728064.0, + "39": 2080728064.0, + "40": 2080728064.0, + "41": 2080728064.0, + "42": 2080728064.0, + "43": 2080728064.0, + "44": 2080728064.0, + "45": 2080728064.0, + "46": 2080728064.0, + "47": 2080728064.0, + "48": 2080728064.0, + "49": 2080728064.0, + "50": 2080728064.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.55391, - "3": 0.33529, - "4": 0.32348, - "5": 0.32324, - "6": 0.3195, - "7": 0.31915, - "8": 0.32104, - "9": 0.31929, - "10": 0.31715, - "11": 0.31704, - "12": 0.3183, - "13": 0.31854, - "14": 0.31648, - "15": 0.31727, - "16": 0.31757, - "17": 0.31624, - "18": 0.31684, - "19": 0.31673, - "20": 0.31934, - "21": 0.32597, - "22": 0.31725, - "23": 0.31852, - "24": 0.3185, - "25": 0.31882, - "26": 0.31722, - "27": 0.316, - "28": 0.31769, - "29": 0.31617, - "30": 0.316, - "31": 0.3163, - "32": 0.31773, - "33": 0.31838, - "34": 0.31669, - "35": 0.31644, - "36": 0.31788, - "37": 0.31694, - "38": 0.31758, - "39": 0.31683, - "40": 0.31834, - "41": 0.31722, - "42": 0.31797, - "43": 0.31796, - "44": 0.31579, - "45": 0.31634, - "46": 0.31549, - "47": 0.31736, - "48": 0.31697, - "49": 0.3171, - "50": 0.31677 + "2": 4.60408, + "3": 0.35515, + "4": 0.33846, + "5": 0.34033, + "6": 0.33894, + "7": 0.33676, + "8": 0.33695, + "9": 0.33815, + "10": 0.33699, + "11": 0.33758, + "12": 0.3366, + "13": 0.3369, + "14": 0.33751, + "15": 0.33827, + "16": 0.33671, + "17": 0.33666, + "18": 0.33682, + "19": 0.33645, + "20": 0.33713, + "21": 0.33677, + "22": 0.33731, + "23": 0.33746, + "24": 0.33652, + "25": 0.33646, + "26": 0.33606, + "27": 0.33592, + "28": 0.33967, + "29": 0.33625, + "30": 0.33608, + "31": 0.33667, + "32": 0.3362, + "33": 0.33564, + "34": 0.33676, + "35": 0.33661, + "36": 0.33565, + "37": 0.33576, + "38": 0.3363, + "39": 0.33673, + "40": 0.33639, + "41": 0.33584, + "42": 0.33677, + "43": 0.33659, + "44": 0.3349, + "45": 0.33524, + "46": 0.33522, + "47": 0.33528, + "48": 0.33594, + "49": 0.33474, + "50": 0.3341 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index dad733298c3..85ee6e3e513 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89855, - "2": 10.90014, - "3": 10.89222, - "4": 10.89178, - "5": 10.89793, + "1": 10.89842, + "2": 10.89984, + "3": 10.89302, + "4": 10.89222, + "5": 10.89805, "6": 10.88299, - "7": 10.88883, - "8": 10.89271, - "9": 10.89251, - "10": 10.8954, - "11": 10.8887, - "12": 10.88607, - "13": 10.88299, - "14": 10.87485, - "15": 10.86721, - "16": 10.85036, - "17": 10.85769, - "18": 10.84317, - "19": 10.85388, - "20": 10.78769, - "21": 10.77727, - "22": 10.77429, - "23": 10.76118, - "24": 10.73113, - "25": 10.72143, - "26": 10.71973, - "27": 10.70153, - "28": 10.64215, - "29": 10.61062, - "30": 10.59331, + "7": 10.88844, + "8": 10.89275, + "9": 10.8927, + "10": 10.89479, + "11": 10.88772, + "12": 10.88616, + "13": 10.88296, + "14": 10.87492, + "15": 10.86635, + "16": 10.85026, + "17": 10.85754, + "18": 10.84381, + "19": 10.85464, + "20": 10.78787, + "21": 10.77781, + "22": 10.77446, + "23": 10.7613, + "24": 10.73152, + "25": 10.72157, + "26": 10.72003, + "27": 10.70155, + "28": 10.64166, + "29": 10.61095, + "30": 10.59234, "31": 10.57783, - "32": 10.55859, - "33": 10.52393, - "34": 10.51223, - "35": 10.4963, - "36": 10.48639, - "37": 10.45372, - "38": 10.4619, - "39": 10.42991, - "40": 10.41153, - "41": 10.3955, - "42": 10.36879, - "43": 10.35083, - "44": 10.3237, - "45": 10.33555, - "46": 10.29334, - "47": 10.28105, - "48": 10.23723, - "49": 10.24644, - "50": 10.23541 + "32": 10.55928, + "33": 10.52447, + "34": 10.51226, + "35": 10.49693, + "36": 10.48657, + "37": 10.45357, + "38": 10.46163, + "39": 10.42974, + "40": 10.41128, + "41": 10.3959, + "42": 10.36906, + "43": 10.35092, + "44": 10.32343, + "45": 10.33546, + "46": 10.29381, + "47": 10.28133, + "48": 10.23785, + "49": 10.24676, + "50": 10.23623 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 15813.0, - "2": 15756.0, - "3": 15574.0, - "4": 15593.0, - "5": 15442.0, - "6": 15296.0, - "7": 15944.0, - "8": 15391.0, - "9": 15934.0, - "10": 15985.0, - "11": 15399.0, - "12": 15252.0, - "13": 15795.0, - "14": 15988.0, - "15": 15334.0, - "16": 15016.0, - "17": 15897.0, - "18": 15798.0, - "19": 15911.0, - "20": 15643.0, - "21": 15256.0, - "22": 15528.0, - "23": 15571.0, - "24": 16175.0, - "25": 14879.0, - "26": 16075.0, - "27": 15490.0, - "28": 15402.0, - "29": 15782.0, - "30": 15398.0, - "31": 15827.0, - "32": 16185.0, - "33": 16174.0, - "34": 16029.0, - "35": 16598.0, - "36": 15849.0, - "37": 16495.0, - "38": 15829.0, - "39": 16972.0, - "40": 17053.0, - "41": 16302.0, - "42": 16225.0, - "43": 17685.0, - "44": 16814.0, - "45": 18616.0, - "46": 17626.0, - "47": 18057.0, - "48": 18789.0, - "49": 20532.0, - "50": 18950.0 + "1": 15620.0, + "2": 15986.0, + "3": 15761.0, + "4": 15774.0, + "5": 15335.0, + "6": 15244.0, + "7": 15809.0, + "8": 15219.0, + "9": 15710.0, + "10": 16019.0, + "11": 15723.0, + "12": 15095.0, + "13": 15669.0, + "14": 15761.0, + "15": 15201.0, + "16": 15269.0, + "17": 15843.0, + "18": 15928.0, + "19": 15871.0, + "20": 15667.0, + "21": 15501.0, + "22": 15619.0, + "23": 15820.0, + "24": 15841.0, + "25": 15040.0, + "26": 15714.0, + "27": 15416.0, + "28": 15566.0, + "29": 15586.0, + "30": 15394.0, + "31": 15755.0, + "32": 16063.0, + "33": 15930.0, + "34": 15945.0, + "35": 16443.0, + "36": 15888.0, + "37": 16158.0, + "38": 15774.0, + "39": 16738.0, + "40": 16971.0, + "41": 16352.0, + "42": 16318.0, + "43": 17671.0, + "44": 16604.0, + "45": 18323.0, + "46": 17918.0, + "47": 17975.0, + "48": 18914.0, + "49": 20266.0, + "50": 18720.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 659367424.0, - "2": 659415552.0, - "3": 659399680.0, - "4": 659349504.0, - "5": 659347456.0, - "6": 659356672.0, - "7": 659391488.0, - "8": 659369984.0, - "9": 659375616.0, - "10": 659511296.0, - "11": 659349504.0, - "12": 659374592.0, - "13": 659357696.0, - "14": 659347968.0, - "15": 659355136.0, - "16": 659369984.0, - "17": 659361280.0, - "18": 659368448.0, - "19": 659379200.0, - "20": 659369984.0, - "21": 659371008.0, - "22": 659374592.0, - "23": 659405312.0, - "24": 659399168.0, - "25": 659346432.0, - "26": 659364864.0, - "27": 659363840.0, - "28": 659347456.0, - "29": 659382784.0, - "30": 659397120.0, - "31": 659468288.0, - "32": 659361280.0, - "33": 659386368.0, - "34": 659367936.0, - "35": 659345408.0, - "36": 659551232.0, - "37": 659373056.0, - "38": 659366400.0, - "39": 659359744.0, - "40": 659350016.0, - "41": 659415552.0, - "42": 659352064.0, - "43": 659392512.0, - "44": 659463680.0, - "45": 659354624.0, - "46": 659469824.0, - "47": 659421696.0, - "48": 659349504.0, - "49": 659426304.0, - "50": 659428352.0 + "1": 659439104.0, + "2": 659481088.0, + "3": 659502080.0, + "4": 659428352.0, + "5": 659430400.0, + "6": 659415040.0, + "7": 659451392.0, + "8": 659438592.0, + "9": 659425280.0, + "10": 659442176.0, + "11": 659412992.0, + "12": 659431936.0, + "13": 659432448.0, + "14": 659432960.0, + "15": 659524096.0, + "16": 659438592.0, + "17": 659414016.0, + "18": 660136448.0, + "19": 659442688.0, + "20": 659418624.0, + "21": 659413504.0, + "22": 659919872.0, + "23": 659412992.0, + "24": 659448320.0, + "25": 659407872.0, + "26": 659604992.0, + "27": 659447296.0, + "28": 660154368.0, + "29": 659434496.0, + "30": 659445760.0, + "31": 659923968.0, + "32": 659480064.0, + "33": 659409920.0, + "34": 659439616.0, + "35": 659409408.0, + "36": 659428864.0, + "37": 659410432.0, + "38": 659426304.0, + "39": 659411456.0, + "40": 659436544.0, + "41": 659472896.0, + "42": 660295168.0, + "43": 659449856.0, + "44": 659676672.0, + "45": 659410432.0, + "46": 659873280.0, + "47": 659778560.0, + "48": 659733504.0, + "49": 659573248.0, + "50": 659980800.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1877530624.0, - "2": 2108462592.0, - "3": 2108938752.0, - "4": 2108938752.0, - "5": 2108938752.0, - "6": 2109357056.0, - "7": 2109357056.0, - "8": 2109357056.0, - "9": 2110007296.0, - "10": 2110007296.0, - "11": 2110007296.0, - "12": 2110007296.0, - "13": 2110007296.0, - "14": 2110007296.0, - "15": 2110007296.0, - "16": 2110007296.0, - "17": 2110007296.0, - "18": 2110077440.0, - "19": 2110077440.0, - "20": 2110077440.0, - "21": 2110077440.0, - "22": 2110077440.0, - "23": 2110077440.0, - "24": 2110077440.0, - "25": 2110077440.0, - "26": 2110077440.0, - "27": 2110077440.0, - "28": 2110077440.0, - "29": 2110077440.0, - "30": 2110077440.0, - "31": 2110077440.0, - "32": 2110077440.0, - "33": 2110077440.0, - "34": 2110077440.0, - "35": 2110077440.0, - "36": 2110077440.0, - "37": 2110077440.0, - "38": 2110077440.0, - "39": 2110203904.0, - "40": 2110203904.0, - "41": 2110203904.0, - "42": 2110203904.0, - "43": 2110519296.0, - "44": 2110519296.0, - "45": 2110519296.0, - "46": 2110983168.0, - "47": 2110983168.0, - "48": 2110983168.0, - "49": 2110983168.0, - "50": 2110983168.0 + "1": 1878177792.0, + "2": 2110117376.0, + "3": 2110117376.0, + "4": 2110137344.0, + "5": 2110137344.0, + "6": 2110137344.0, + "7": 2110137344.0, + "8": 2110137344.0, + "9": 2111263744.0, + "10": 2111263744.0, + "11": 2111263744.0, + "12": 2111263744.0, + "13": 2111263744.0, + "14": 2111263744.0, + "15": 2111263744.0, + "16": 2111263744.0, + "17": 2111263744.0, + "18": 2111337984.0, + "19": 2111337984.0, + "20": 2111337984.0, + "21": 2111337984.0, + "22": 2111337984.0, + "23": 2111919104.0, + "24": 2111919104.0, + "25": 2111919104.0, + "26": 2111919104.0, + "27": 2111919104.0, + "28": 2111919104.0, + "29": 2111919104.0, + "30": 2111919104.0, + "31": 2111919104.0, + "32": 2111919104.0, + "33": 2111919104.0, + "34": 2111919104.0, + "35": 2111919104.0, + "36": 2111919104.0, + "37": 2111919104.0, + "38": 2111919104.0, + "39": 2111919104.0, + "40": 2111919104.0, + "41": 2111919104.0, + "42": 2111919104.0, + "43": 2111919104.0, + "44": 2111919104.0, + "45": 2111919104.0, + "46": 2111919104.0, + "47": 2111919104.0, + "48": 2111919104.0, + "49": 2111919104.0, + "50": 2111959040.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.05664, - "3": 0.42119, - "4": 0.39552, - "5": 0.39405, - "6": 0.39985, - "7": 0.39889, - "8": 0.39776, - "9": 0.40046, - "10": 0.39845, - "11": 0.39958, - "12": 0.39391, - "13": 0.39121, - "14": 0.39395, - "15": 0.39142, - "16": 0.39482, - "17": 0.39023, - "18": 0.39105, - "19": 0.38986, - "20": 0.38396, - "21": 0.39066, - "22": 0.39781, - "23": 0.3963, - "24": 0.39583, - "25": 0.39535, - "26": 0.39431, - "27": 0.39054, - "28": 0.3976, - "29": 0.39148, - "30": 0.39109, - "31": 0.39185, - "32": 0.39088, - "33": 0.38971, - "34": 0.38923, - "35": 0.38913, - "36": 0.38094, - "37": 0.38687, - "38": 0.38996, - "39": 0.38992, - "40": 0.38642, - "41": 0.39137, - "42": 0.38922, - "43": 0.39267, - "44": 0.3892, - "45": 0.38995, - "46": 0.39164, - "47": 0.39271, - "48": 0.39196, - "49": 0.38853, - "50": 0.39357 + "2": 11.23151, + "3": 0.38679, + "4": 0.32897, + "5": 0.33041, + "6": 0.32275, + "7": 0.32517, + "8": 0.32157, + "9": 0.31944, + "10": 0.32307, + "11": 0.33106, + "12": 0.31843, + "13": 0.31651, + "14": 0.3171, + "15": 0.31833, + "16": 0.32056, + "17": 0.31472, + "18": 0.31877, + "19": 0.31624, + "20": 0.31455, + "21": 0.31455, + "22": 0.31417, + "23": 0.31433, + "24": 0.31906, + "25": 0.31624, + "26": 0.31579, + "27": 0.3127, + "28": 0.31239, + "29": 0.31617, + "30": 0.3153, + "31": 0.3157, + "32": 0.31627, + "33": 0.31409, + "34": 0.31439, + "35": 0.31877, + "36": 0.31449, + "37": 0.31653, + "38": 0.31602, + "39": 0.31515, + "40": 0.31607, + "41": 0.31491, + "42": 0.31403, + "43": 0.31876, + "44": 0.31362, + "45": 0.3137, + "46": 0.31546, + "47": 0.31246, + "48": 0.31526, + "49": 0.31872, + "50": 0.31573 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json index 74330a978e2..b33c81893f7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93828, - "2": 10.92576, - "3": 10.93494, - "4": 10.93745, + "1": 10.93861, + "2": 10.92537, + "3": 10.9346, + "4": 10.93786, "5": 10.93093, - "6": 10.93372, - "7": 10.93833, - "8": 10.92805, - "9": 10.92631, - "10": 10.92854, - "11": 10.92345, - "12": 10.93211, - "13": 10.90542, - "14": 10.90387, - "15": 10.90247, - "16": 10.88246, - "17": 10.88458, - "18": 10.88268, - "19": 10.87497, - "20": 10.8139, - "21": 10.806, - "22": 10.79346, - "23": 10.79829, - "24": 10.77081, - "25": 10.76825, - "26": 10.75015, - "27": 10.73202, - "28": 10.66464, - "29": 10.63113, - "30": 10.61108, - "31": 10.60748, - "32": 10.58969, - "33": 10.56299, - "34": 10.52678, - "35": 10.5296, - "36": 10.5035, - "37": 10.48112, - "38": 10.48478, - "39": 10.45202, - "40": 10.42947, - "41": 10.41208, - "42": 10.39673, - "43": 10.37544, - "44": 10.35435, + "6": 10.93403, + "7": 10.93809, + "8": 10.92787, + "9": 10.92646, + "10": 10.92835, + "11": 10.92266, + "12": 10.93279, + "13": 10.90541, + "14": 10.90432, + "15": 10.90226, + "16": 10.88178, + "17": 10.88414, + "18": 10.88298, + "19": 10.87464, + "20": 10.81406, + "21": 10.80579, + "22": 10.79285, + "23": 10.79827, + "24": 10.77026, + "25": 10.76833, + "26": 10.7507, + "27": 10.73201, + "28": 10.66572, + "29": 10.63124, + "30": 10.61187, + "31": 10.60737, + "32": 10.58938, + "33": 10.56312, + "34": 10.52692, + "35": 10.5287, + "36": 10.50346, + "37": 10.48189, + "38": 10.48513, + "39": 10.45172, + "40": 10.42975, + "41": 10.41232, + "42": 10.39718, + "43": 10.37503, + "44": 10.35419, "45": 10.35117, - "46": 10.32051, - "47": 10.3055, - "48": 10.26287, - "49": 10.25801, - "50": 10.25626 + "46": 10.32015, + "47": 10.30603, + "48": 10.26275, + "49": 10.25834, + "50": 10.25658 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 15594.0, - "2": 15896.0, - "3": 15740.0, - "4": 15933.0, - "5": 15659.0, - "6": 15220.0, - "7": 15998.0, - "8": 15231.0, - "9": 15470.0, - "10": 16088.0, - "11": 15809.0, - "12": 15315.0, - "13": 15890.0, - "14": 15760.0, - "15": 15259.0, - "16": 15304.0, - "17": 15836.0, - "18": 15921.0, - "19": 15656.0, - "20": 15734.0, - "21": 15424.0, - "22": 16072.0, - "23": 15586.0, - "24": 16338.0, - "25": 14747.0, - "26": 15909.0, - "27": 15945.0, - "28": 15767.0, - "29": 15714.0, - "30": 15770.0, - "31": 16164.0, - "32": 16277.0, - "33": 16281.0, - "34": 16107.0, - "35": 16509.0, - "36": 16198.0, - "37": 16504.0, - "38": 15708.0, - "39": 16701.0, - "40": 17411.0, - "41": 16360.0, - "42": 16329.0, - "43": 16729.0, - "44": 16700.0, - "45": 18790.0, - "46": 17776.0, - "47": 17344.0, - "48": 18096.0, - "49": 20474.0, - "50": 18027.0 + "1": 15184.0, + "2": 15900.0, + "3": 15676.0, + "4": 16071.0, + "5": 15591.0, + "6": 14981.0, + "7": 15865.0, + "8": 15410.0, + "9": 15733.0, + "10": 15654.0, + "11": 15658.0, + "12": 15132.0, + "13": 15939.0, + "14": 15757.0, + "15": 15235.0, + "16": 15119.0, + "17": 15508.0, + "18": 15898.0, + "19": 15863.0, + "20": 15682.0, + "21": 15610.0, + "22": 16064.0, + "23": 15495.0, + "24": 16307.0, + "25": 15029.0, + "26": 15603.0, + "27": 16135.0, + "28": 15426.0, + "29": 15479.0, + "30": 16122.0, + "31": 16211.0, + "32": 15945.0, + "33": 15926.0, + "34": 16012.0, + "35": 16282.0, + "36": 15861.0, + "37": 16556.0, + "38": 15901.0, + "39": 16676.0, + "40": 17252.0, + "41": 16451.0, + "42": 16280.0, + "43": 16670.0, + "44": 16474.0, + "45": 18712.0, + "46": 17880.0, + "47": 17886.0, + "48": 18288.0, + "49": 20228.0, + "50": 18150.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 660973056.0, - "2": 660945920.0, - "3": 661182976.0, - "4": 660915200.0, - "5": 661140480.0, - "6": 660893696.0, - "7": 660894208.0, - "8": 660989440.0, - "9": 660927488.0, - "10": 660937728.0, - "11": 660974592.0, - "12": 660926464.0, - "13": 660894720.0, - "14": 660917760.0, - "15": 660934144.0, - "16": 660894720.0, - "17": 660892672.0, - "18": 660899840.0, - "19": 660894208.0, - "20": 660967936.0, - "21": 660930048.0, - "22": 660918784.0, - "23": 660901376.0, - "24": 660923392.0, - "25": 660893184.0, - "26": 660912640.0, - "27": 660894208.0, - "28": 660948992.0, - "29": 661011456.0, - "30": 660893696.0, - "31": 660909568.0, - "32": 660955648.0, - "33": 660906496.0, - "34": 660892672.0, - "35": 660981760.0, - "36": 661134848.0, - "37": 660935680.0, - "38": 660893696.0, - "39": 660891136.0, - "40": 661140992.0, - "41": 660901888.0, - "42": 660890624.0, - "43": 661062656.0, - "44": 660900352.0, - "45": 661000704.0, - "46": 661164544.0, - "47": 660891648.0, - "48": 660947968.0, - "49": 660892160.0, - "50": 660937216.0 + "1": 659063808.0, + "2": 659015680.0, + "3": 659014656.0, + "4": 659013632.0, + "5": 659053568.0, + "6": 659087872.0, + "7": 659015680.0, + "8": 659037696.0, + "9": 659013632.0, + "10": 659073024.0, + "11": 659229184.0, + "12": 659014656.0, + "13": 659101184.0, + "14": 659013632.0, + "15": 659014144.0, + "16": 659014656.0, + "17": 659128320.0, + "18": 659014144.0, + "19": 659014144.0, + "20": 659017216.0, + "21": 659011584.0, + "22": 659024896.0, + "23": 659277824.0, + "24": 659013632.0, + "25": 659013632.0, + "26": 659037184.0, + "27": 659014144.0, + "28": 659223040.0, + "29": 659218432.0, + "30": 659014144.0, + "31": 659034624.0, + "32": 659015168.0, + "33": 659019264.0, + "34": 659038208.0, + "35": 659015680.0, + "36": 659274240.0, + "37": 659048960.0, + "38": 659014656.0, + "39": 659261440.0, + "40": 659015168.0, + "41": 659070464.0, + "42": 659019264.0, + "43": 659013632.0, + "44": 659013120.0, + "45": 659038208.0, + "46": 659013120.0, + "47": 659013120.0, + "48": 659065344.0, + "49": 659013632.0, + "50": 659060736.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1877622272.0, - "2": 2110339072.0, - "3": 2110339072.0, - "4": 2110480896.0, - "5": 2112296448.0, - "6": 2112296448.0, - "7": 2112296448.0, - "8": 2112296448.0, - "9": 2112296448.0, - "10": 2112296448.0, - "11": 2112296448.0, - "12": 2112296448.0, - "13": 2112296448.0, - "14": 2112296448.0, - "15": 2112296448.0, - "16": 2112296448.0, - "17": 2112296448.0, - "18": 2112296448.0, - "19": 2112296448.0, - "20": 2112296448.0, - "21": 2112296448.0, - "22": 2112296448.0, - "23": 2112438784.0, - "24": 2112438784.0, - "25": 2112438784.0, - "26": 2112438784.0, - "27": 2112438784.0, - "28": 2112438784.0, - "29": 2112438784.0, - "30": 2112438784.0, - "31": 2112438784.0, - "32": 2112438784.0, - "33": 2112438784.0, - "34": 2112438784.0, - "35": 2112438784.0, - "36": 2112438784.0, - "37": 2112438784.0, - "38": 2112438784.0, - "39": 2112438784.0, - "40": 2112438784.0, - "41": 2112438784.0, - "42": 2112438784.0, - "43": 2112438784.0, - "44": 2112438784.0, - "45": 2112438784.0, - "46": 2112438784.0, - "47": 2112438784.0, - "48": 2112438784.0, - "49": 2112438784.0, - "50": 2112438784.0 + "1": 1877660672.0, + "2": 2108270592.0, + "3": 2108270592.0, + "4": 2108270592.0, + "5": 2108640768.0, + "6": 2108914688.0, + "7": 2108914688.0, + "8": 2108914688.0, + "9": 2108914688.0, + "10": 2108914688.0, + "11": 2108914688.0, + "12": 2108914688.0, + "13": 2109423104.0, + "14": 2109423104.0, + "15": 2109423104.0, + "16": 2109423104.0, + "17": 2109770752.0, + "18": 2109770752.0, + "19": 2109770752.0, + "20": 2110347776.0, + "21": 2110347776.0, + "22": 2110347776.0, + "23": 2110347776.0, + "24": 2110347776.0, + "25": 2110347776.0, + "26": 2110347776.0, + "27": 2110347776.0, + "28": 2110347776.0, + "29": 2110347776.0, + "30": 2110347776.0, + "31": 2110347776.0, + "32": 2110347776.0, + "33": 2110347776.0, + "34": 2110347776.0, + "35": 2110856192.0, + "36": 2110856192.0, + "37": 2110856192.0, + "38": 2110856192.0, + "39": 2110856192.0, + "40": 2110856192.0, + "41": 2110856192.0, + "42": 2110856192.0, + "43": 2110856192.0, + "44": 2110856192.0, + "45": 2110856192.0, + "46": 2110856192.0, + "47": 2110856192.0, + "48": 2110856192.0, + "49": 2110856192.0, + "50": 2110856192.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.10018, - "3": 0.27822, - "4": 0.24824, - "5": 0.25089, - "6": 0.24011, - "7": 0.23435, - "8": 0.2318, - "9": 0.22995, - "10": 0.24267, - "11": 0.22511, - "12": 0.22522, - "13": 0.22541, - "14": 0.22816, - "15": 0.22377, - "16": 0.22503, - "17": 0.23893, - "18": 0.22526, - "19": 0.22884, - "20": 0.23311, - "21": 0.22811, - "22": 0.22634, - "23": 0.22985, - "24": 0.22727, - "25": 0.22692, - "26": 0.23024, - "27": 0.22368, - "28": 0.2251, - "29": 0.22639, - "30": 0.22497, - "31": 0.23006, - "32": 0.23763, - "33": 0.22423, - "34": 0.23301, - "35": 0.22628, - "36": 0.22382, - "37": 0.22185, - "38": 0.22466, - "39": 0.22917, - "40": 0.22908, - "41": 0.22229, - "42": 0.2207, - "43": 0.22921, - "44": 0.22128, - "45": 0.22492, - "46": 0.22189, - "47": 0.23987, - "48": 0.22274, - "49": 0.22922, - "50": 0.22277 + "2": 7.95392, + "3": 0.29083, + "4": 0.25573, + "5": 0.25253, + "6": 0.26376, + "7": 0.25325, + "8": 0.24535, + "9": 0.24492, + "10": 0.24217, + "11": 0.24649, + "12": 0.24076, + "13": 0.24173, + "14": 0.24022, + "15": 0.23919, + "16": 0.24131, + "17": 0.24319, + "18": 0.24401, + "19": 0.24333, + "20": 0.24505, + "21": 0.26178, + "22": 0.23923, + "23": 0.23808, + "24": 0.24104, + "25": 0.24657, + "26": 0.23709, + "27": 0.2354, + "28": 0.23767, + "29": 0.25518, + "30": 0.23641, + "31": 0.23692, + "32": 0.2342, + "33": 0.23741, + "34": 0.23491, + "35": 0.24422, + "36": 0.23849, + "37": 0.23568, + "38": 0.23943, + "39": 0.23681, + "40": 0.23743, + "41": 0.23644, + "42": 0.23713, + "43": 0.24757, + "44": 0.2363, + "45": 0.2391, + "46": 0.24071, + "47": 0.24818, + "48": 0.23345, + "49": 0.24237, + "50": 0.24418 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json index 4c8fb5d696b..bc80b167114 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90655, - "4": 10.90577, - "5": 10.91353, - "6": 10.8974, - "7": 10.89795, - "8": 10.91315, - "9": 10.89647, - "10": 10.90362, - "11": 10.90058, - "12": 10.91863, - "13": 10.88451, - "14": 10.87468, - "15": 10.90997, - "16": 10.90304, - "17": 10.88702, - "18": 10.89383, - "19": 10.89477, - "20": 10.89108, - "21": 10.88732, - "22": 10.90287, - "23": 10.91197, - "24": 10.88563, - "25": 10.90095, - "26": 10.88886, - "27": 10.89671, - "28": 10.88189, - "29": 10.88993, - "30": 10.91034, - "31": 10.89135, - "32": 10.8888, - "33": 10.8978, - "34": 10.87277, - "35": 10.89612, - "36": 10.90758, - "37": 10.8709, - "38": 10.87877, - "39": 10.88567, - "40": 10.89273, - "41": 10.88396, - "42": 10.89532, - "43": 10.87994, - "44": 10.88499, - "45": 10.88349, - "46": 10.88634, - "47": 10.88604, - "48": 10.86357, - "49": 10.87627, - "50": 10.88323, - "51": 10.8932, - "52": 10.87074, - "53": 10.85901, - "54": 10.8708, - "55": 10.86805, - "56": 10.87369, - "57": 10.84455, - "58": 10.85609, - "59": 10.84667, - "60": 10.84361, - "61": 10.86077, - "62": 10.85729, - "63": 10.86045, - "64": 10.83869, - "65": 10.82777, - "66": 10.84578, - "67": 10.8274, - "68": 10.82991, - "69": 10.81829, - "70": 10.82845, - "71": 10.82632, - "72": 10.80841, - "73": 10.80779, - "74": 10.80552, - "75": 10.81455, - "76": 10.81142, - "77": 10.80752, - "78": 10.79152, - "79": 10.80266, - "80": 10.78868, - "81": 10.79558, - "82": 10.79566, - "83": 10.78578, - "84": 10.75723, - "85": 10.76273, - "86": 10.7768, - "87": 10.79587, - "88": 10.7746, - "89": 10.77528, - "90": 10.76475, - "91": 10.74024, - "92": 10.75993, - "93": 10.7468, - "94": 10.73446, - "95": 10.75268, - "96": 10.7234, - "97": 10.71513, - "98": 10.72592, - "99": 10.74621, - "100": 10.69564 + "1": 10.90111, + "2": 10.8957, + "3": 10.90636, + "4": 10.90606, + "5": 10.91349, + "6": 10.89747, + "7": 10.89834, + "8": 10.91366, + "9": 10.89559, + "10": 10.90375, + "11": 10.90014, + "12": 10.91887, + "13": 10.88393, + "14": 10.87584, + "15": 10.91035, + "16": 10.90325, + "17": 10.88717, + "18": 10.89416, + "19": 10.89487, + "20": 10.89092, + "21": 10.88636, + "22": 10.90341, + "23": 10.91256, + "24": 10.88547, + "25": 10.90164, + "26": 10.88848, + "27": 10.89705, + "28": 10.88168, + "29": 10.88997, + "30": 10.91097, + "31": 10.89128, + "32": 10.88933, + "33": 10.89772, + "34": 10.87339, + "35": 10.89595, + "36": 10.90751, + "37": 10.87112, + "38": 10.87839, + "39": 10.88579, + "40": 10.89247, + "41": 10.88364, + "42": 10.89496, + "43": 10.88058, + "44": 10.88492, + "45": 10.88361, + "46": 10.88582, + "47": 10.88565, + "48": 10.86424, + "49": 10.8759, + "50": 10.88259, + "51": 10.89305, + "52": 10.87123, + "53": 10.85905, + "54": 10.8702, + "55": 10.86807, + "56": 10.87324, + "57": 10.84465, + "58": 10.8563, + "59": 10.84628, + "60": 10.84343, + "61": 10.8604, + "62": 10.85675, + "63": 10.86117, + "64": 10.83852, + "65": 10.82804, + "66": 10.84584, + "67": 10.82846, + "68": 10.83007, + "69": 10.81894, + "70": 10.82797, + "71": 10.82677, + "72": 10.8088, + "73": 10.80753, + "74": 10.80659, + "75": 10.81472, + "76": 10.81086, + "77": 10.80817, + "78": 10.79255, + "79": 10.80218, + "80": 10.7879, + "81": 10.79487, + "82": 10.79556, + "83": 10.78629, + "84": 10.75658, + "85": 10.76241, + "86": 10.77699, + "87": 10.79624, + "88": 10.77534, + "89": 10.77559, + "90": 10.76414, + "91": 10.74033, + "92": 10.76012, + "93": 10.74683, + "94": 10.73435, + "95": 10.75231, + "96": 10.72344, + "97": 10.71428, + "98": 10.72631, + "99": 10.74637, + "100": 10.69592 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1431.0, - "4": 1283.0, - "5": 1254.0, - "6": 1261.0, - "7": 1295.0, - "8": 1160.0, - "9": 1242.0, - "10": 1143.0, - "11": 1217.0, - "12": 1226.0, - "13": 1325.0, - "14": 1163.0, - "15": 1218.0, - "16": 1232.0, - "17": 1256.0, - "18": 1128.0, - "19": 1231.0, - "20": 1137.0, - "21": 1199.0, - "22": 1215.0, - "23": 1278.0, - "24": 1179.0, - "25": 1273.0, - "26": 1237.0, - "27": 1316.0, - "28": 1183.0, - "29": 1171.0, - "30": 1295.0, - "31": 1254.0, - "32": 1258.0, - "33": 1212.0, - "34": 1220.0, - "35": 1350.0, - "36": 1214.0, - "37": 1213.0, - "38": 1259.0, - "39": 1382.0, - "40": 1357.0, - "41": 1252.0, - "42": 1115.0, - "43": 1180.0, - "44": 1237.0, - "45": 1170.0, - "46": 1438.0, - "47": 1193.0, - "48": 1124.0, - "49": 1338.0, - "50": 1377.0, - "51": 1277.0, - "52": 1234.0, - "53": 1204.0, - "54": 1132.0, - "55": 1138.0, - "56": 1147.0, - "57": 1127.0, - "58": 1290.0, - "59": 1074.0, - "60": 1286.0, - "61": 1259.0, - "62": 1179.0, - "63": 1243.0, - "64": 1322.0, - "65": 1162.0, - "66": 1182.0, - "67": 1248.0, - "68": 1276.0, - "69": 1154.0, - "70": 1173.0, - "71": 1122.0, - "72": 1236.0, - "73": 1199.0, - "74": 1354.0, - "75": 1322.0, - "76": 1301.0, - "77": 1168.0, - "78": 1294.0, - "79": 1295.0, - "80": 1216.0, - "81": 1305.0, - "82": 1201.0, - "83": 1196.0, - "84": 1150.0, - "85": 1246.0, - "86": 1309.0, - "87": 1133.0, - "88": 1166.0, - "89": 1146.0, - "90": 1389.0, - "91": 1246.0, - "92": 1195.0, - "93": 1286.0, - "94": 1318.0, - "95": 1040.0, - "96": 1248.0, - "97": 1192.0, - "98": 1376.0, - "99": 1170.0, - "100": 1292.0 + "1": 1133.0, + "2": 1120.0, + "3": 1371.0, + "4": 1297.0, + "5": 1158.0, + "6": 1283.0, + "7": 1334.0, + "8": 1122.0, + "9": 1269.0, + "10": 1134.0, + "11": 1184.0, + "12": 1223.0, + "13": 1340.0, + "14": 1232.0, + "15": 1234.0, + "16": 1179.0, + "17": 1212.0, + "18": 1083.0, + "19": 1187.0, + "20": 1174.0, + "21": 1196.0, + "22": 1227.0, + "23": 1318.0, + "24": 1113.0, + "25": 1191.0, + "26": 1243.0, + "27": 1343.0, + "28": 1251.0, + "29": 1186.0, + "30": 1221.0, + "31": 1205.0, + "32": 1269.0, + "33": 1254.0, + "34": 1196.0, + "35": 1308.0, + "36": 1277.0, + "37": 1196.0, + "38": 1321.0, + "39": 1407.0, + "40": 1299.0, + "41": 1284.0, + "42": 1170.0, + "43": 1170.0, + "44": 1252.0, + "45": 1154.0, + "46": 1403.0, + "47": 1148.0, + "48": 1176.0, + "49": 1389.0, + "50": 1329.0, + "51": 1209.0, + "52": 1224.0, + "53": 1276.0, + "54": 1198.0, + "55": 1179.0, + "56": 1166.0, + "57": 1168.0, + "58": 1318.0, + "59": 1047.0, + "60": 1118.0, + "61": 1192.0, + "62": 1108.0, + "63": 1225.0, + "64": 1256.0, + "65": 1221.0, + "66": 1228.0, + "67": 1193.0, + "68": 1279.0, + "69": 1141.0, + "70": 1248.0, + "71": 1165.0, + "72": 1230.0, + "73": 1236.0, + "74": 1336.0, + "75": 1346.0, + "76": 1259.0, + "77": 1205.0, + "78": 1217.0, + "79": 1298.0, + "80": 1228.0, + "81": 1335.0, + "82": 1198.0, + "83": 1246.0, + "84": 1152.0, + "85": 1250.0, + "86": 1287.0, + "87": 1111.0, + "88": 1169.0, + "89": 1119.0, + "90": 1402.0, + "91": 1297.0, + "92": 1194.0, + "93": 1271.0, + "94": 1409.0, + "95": 980.0, + "96": 1170.0, + "97": 1212.0, + "98": 1325.0, + "99": 1125.0, + "100": 1206.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 994062848.0, - "2": 994053120.0, + "1": 994063360.0, + "2": 994054144.0, "3": 994029056.0, - "4": 994009600.0, - "5": 994060288.0, - "6": 994043904.0, - "7": 993970688.0, - "8": 994005504.0, - "9": 994008576.0, - "10": 994040832.0, - "11": 994017280.0, - "12": 994023936.0, - "13": 994028032.0, - "14": 994029568.0, - "15": 994005504.0, + "4": 994010624.0, + "5": 994059776.0, + "6": 994042368.0, + "7": 993970176.0, + "8": 994006528.0, + "9": 994008064.0, + "10": 994039808.0, + "11": 994015232.0, + "12": 994024960.0, + "13": 994029056.0, + "14": 994030592.0, + "15": 994007040.0, "16": 993992704.0, "17": 994001408.0, - "18": 994002944.0, - "19": 994032128.0, - "20": 993999360.0, - "21": 994050560.0, - "22": 994078720.0, - "23": 994051584.0, - "24": 993963520.0, - "25": 994044928.0, - "26": 994075648.0, - "27": 994029056.0, - "28": 993983488.0, - "29": 994057216.0, - "30": 993989632.0, - "31": 993997312.0, - "32": 994005504.0, - "33": 994009088.0, - "34": 993986560.0, - "35": 993973248.0, - "36": 994038272.0, + "18": 994003968.0, + "19": 994032640.0, + "20": 993997312.0, + "21": 994052608.0, + "22": 994075648.0, + "23": 994049024.0, + "24": 993964032.0, + "25": 994045440.0, + "26": 994076672.0, + "27": 994028544.0, + "28": 993984512.0, + "29": 994056704.0, + "30": 993990656.0, + "31": 993996800.0, + "32": 994003456.0, + "33": 994011136.0, + "34": 993989120.0, + "35": 993974784.0, + "36": 994040832.0, "37": 993988608.0, - "38": 994048000.0, - "39": 994056192.0, - "40": 994000896.0, - "41": 994023424.0, - "42": 993964032.0, - "43": 994031104.0, - "44": 994015232.0, - "45": 993988096.0, - "46": 994054144.0, - "47": 994010624.0, + "38": 994046976.0, + "39": 994054144.0, + "40": 994001408.0, + "41": 994021888.0, + "42": 993964544.0, + "43": 994032640.0, + "44": 994014208.0, + "45": 993989120.0, + "46": 994052096.0, + "47": 994011136.0, "48": 994040320.0, - "49": 993998336.0, - "50": 994048000.0, - "51": 994031104.0, - "52": 994000896.0, - "53": 993953792.0, - "54": 994048512.0, - "55": 994041344.0, - "56": 994039296.0, - "57": 994099200.0, - "58": 994036224.0, - "59": 994042880.0, - "60": 994017792.0, - "61": 993987584.0, - "62": 993977344.0, - "63": 993988096.0, - "64": 994016768.0, - "65": 994022400.0, - "66": 994036736.0, - "67": 994019328.0, + "49": 993998848.0, + "50": 994044928.0, + "51": 994030592.0, + "52": 994003968.0, + "53": 993953280.0, + "54": 994051072.0, + "55": 994039808.0, + "56": 994038784.0, + "57": 994098688.0, + "58": 994035200.0, + "59": 994040832.0, + "60": 994019328.0, + "61": 993988608.0, + "62": 993980928.0, + "63": 993987584.0, + "64": 994015744.0, + "65": 994023424.0, + "66": 994037248.0, + "67": 994018816.0, "68": 994020352.0, - "69": 993982464.0, - "70": 994070016.0, - "71": 994048512.0, - "72": 994044928.0, - "73": 993976320.0, - "74": 994013184.0, - "75": 994061824.0, - "76": 994019328.0, - "77": 994062848.0, - "78": 994003968.0, - "79": 994072576.0, + "69": 993985024.0, + "70": 994067456.0, + "71": 994047488.0, + "72": 994044416.0, + "73": 993973760.0, + "74": 994015744.0, + "75": 994060800.0, + "76": 994017792.0, + "77": 994060288.0, + "78": 994006016.0, + "79": 994072064.0, "80": 993993728.0, - "81": 994045952.0, - "82": 993991680.0, - "83": 994055680.0, - "84": 994026496.0, - "85": 994033664.0, - "86": 994017280.0, + "81": 994046464.0, + "82": 993994752.0, + "83": 994054656.0, + "84": 994024448.0, + "85": 994032128.0, + "86": 994016256.0, "87": 994030080.0, - "88": 994014208.0, - "89": 994002432.0, - "90": 994024448.0, - "91": 994012160.0, - "92": 994010624.0, - "93": 994039808.0, - "94": 994013184.0, - "95": 994034688.0, - "96": 994028032.0, - "97": 993995776.0, - "98": 994034176.0, - "99": 994049024.0, - "100": 994081280.0 + "88": 994010112.0, + "89": 994001408.0, + "90": 994024960.0, + "91": 994011136.0, + "92": 994009600.0, + "93": 994040320.0, + "94": 994015232.0, + "95": 994036736.0, + "96": 994028544.0, + "97": 993997312.0, + "98": 994035712.0, + "99": 994051584.0, + "100": 994083328.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3207447552.0, - "2": 3491065344.0, - "3": 3491065344.0, - "4": 3491065344.0, - "5": 3497349632.0, - "6": 3497349632.0, - "7": 3497349632.0, - "8": 3497349632.0, - "9": 3497349632.0, - "10": 3497349632.0, - "11": 3497349632.0, - "12": 3497349632.0, - "13": 3497349632.0, - "14": 3497349632.0, - "15": 3497349632.0, - "16": 3497349632.0, - "17": 3497349632.0, - "18": 3497349632.0, - "19": 3497349632.0, - "20": 3497349632.0, - "21": 3497349632.0, - "22": 3506985472.0, - "23": 3506985472.0, - "24": 3506985472.0, - "25": 3506985472.0, - "26": 3512014336.0, - "27": 3512014336.0, - "28": 3512014336.0, - "29": 3512014336.0, - "30": 3512014336.0, - "31": 3512014336.0, - "32": 3512014336.0, - "33": 3512014336.0, - "34": 3512014336.0, - "35": 3512014336.0, - "36": 3512014336.0, - "37": 3512014336.0, - "38": 3512014336.0, - "39": 3512014336.0, - "40": 3512014336.0, - "41": 3512014336.0, - "42": 3512014336.0, - "43": 3512014336.0, - "44": 3512014336.0, - "45": 3512014336.0, - "46": 3512014336.0, - "47": 3512014336.0, - "48": 3512014336.0, - "49": 3512014336.0, - "50": 3512014336.0, - "51": 3512014336.0, - "52": 3512014336.0, - "53": 3512014336.0, - "54": 3512014336.0, - "55": 3512014336.0, - "56": 3512014336.0, - "57": 3526792704.0, - "58": 3526792704.0, - "59": 3526792704.0, - "60": 3526792704.0, - "61": 3526792704.0, - "62": 3526792704.0, - "63": 3526792704.0, - "64": 3526792704.0, - "65": 3526792704.0, - "66": 3526792704.0, - "67": 3526792704.0, - "68": 3526792704.0, - "69": 3526792704.0, - "70": 3526792704.0, - "71": 3526792704.0, - "72": 3526792704.0, - "73": 3526792704.0, - "74": 3526792704.0, - "75": 3526792704.0, - "76": 3526792704.0, - "77": 3526792704.0, - "78": 3526792704.0, - "79": 3526792704.0, - "80": 3526792704.0, - "81": 3526792704.0, - "82": 3526792704.0, - "83": 3526792704.0, - "84": 3526792704.0, - "85": 3526792704.0, - "86": 3526792704.0, - "87": 3526792704.0, - "88": 3526792704.0, - "89": 3526792704.0, - "90": 3526792704.0, - "91": 3526792704.0, - "92": 3526792704.0, - "93": 3526792704.0, - "94": 3526792704.0, - "95": 3526792704.0, - "96": 3526792704.0, - "97": 3526792704.0, - "98": 3526792704.0, - "99": 3526792704.0, - "100": 3526792704.0 + "1": 3205999104.0, + "2": 3490994688.0, + "3": 3490994688.0, + "4": 3490994688.0, + "5": 3498078208.0, + "6": 3498078208.0, + "7": 3498078208.0, + "8": 3498078208.0, + "9": 3498078208.0, + "10": 3498078208.0, + "11": 3498078208.0, + "12": 3498078208.0, + "13": 3498078208.0, + "14": 3498078208.0, + "15": 3498078208.0, + "16": 3498078208.0, + "17": 3498078208.0, + "18": 3498078208.0, + "19": 3498078208.0, + "20": 3498078208.0, + "21": 3498078208.0, + "22": 3507077632.0, + "23": 3507077632.0, + "24": 3507077632.0, + "25": 3507077632.0, + "26": 3510859264.0, + "27": 3510859264.0, + "28": 3510859264.0, + "29": 3510859264.0, + "30": 3510859264.0, + "31": 3510859264.0, + "32": 3510859264.0, + "33": 3510859264.0, + "34": 3510859264.0, + "35": 3510859264.0, + "36": 3510859264.0, + "37": 3510859264.0, + "38": 3510859264.0, + "39": 3510859264.0, + "40": 3510859264.0, + "41": 3510859264.0, + "42": 3510859264.0, + "43": 3510859264.0, + "44": 3510859264.0, + "45": 3510859264.0, + "46": 3510859264.0, + "47": 3510859264.0, + "48": 3510859264.0, + "49": 3510859264.0, + "50": 3510859264.0, + "51": 3510859264.0, + "52": 3510859264.0, + "53": 3510859264.0, + "54": 3510859264.0, + "55": 3510859264.0, + "56": 3510859264.0, + "57": 3526288384.0, + "58": 3526288384.0, + "59": 3526288384.0, + "60": 3526288384.0, + "61": 3526288384.0, + "62": 3526288384.0, + "63": 3526288384.0, + "64": 3526288384.0, + "65": 3526288384.0, + "66": 3526288384.0, + "67": 3526288384.0, + "68": 3526288384.0, + "69": 3526288384.0, + "70": 3526288384.0, + "71": 3526288384.0, + "72": 3526288384.0, + "73": 3526288384.0, + "74": 3526288384.0, + "75": 3526288384.0, + "76": 3526288384.0, + "77": 3526288384.0, + "78": 3526288384.0, + "79": 3526288384.0, + "80": 3526288384.0, + "81": 3526288384.0, + "82": 3526288384.0, + "83": 3526288384.0, + "84": 3526288384.0, + "85": 3526288384.0, + "86": 3526288384.0, + "87": 3526288384.0, + "88": 3526288384.0, + "89": 3526288384.0, + "90": 3526288384.0, + "91": 3526288384.0, + "92": 3526288384.0, + "93": 3526288384.0, + "94": 3526288384.0, + "95": 3526288384.0, + "96": 3526288384.0, + "97": 3526288384.0, + "98": 3526288384.0, + "99": 3526288384.0, + "100": 3526288384.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.43392, - "3": 0.22954, - "4": 0.2187, - "5": 0.21296, - "6": 0.20548, - "7": 0.20909, - "8": 0.22033, - "9": 0.20266, - "10": 0.20368, - "11": 0.20814, - "12": 0.19574, - "13": 0.19885, - "14": 0.19615, - "15": 0.20178, - "16": 0.19024, - "17": 0.19778, - "18": 0.19123, - "19": 0.19898, - "20": 0.19976, - "21": 0.18857, - "22": 0.19138, - "23": 0.19297, - "24": 0.19637, - "25": 0.19458, - "26": 0.19364, - "27": 0.1858, - "28": 0.17947, - "29": 0.19539, - "30": 0.19752, - "31": 0.18516, - "32": 0.19246, - "33": 0.19096, - "34": 0.2006, - "35": 0.19289, - "36": 0.18703, - "37": 0.20608, - "38": 0.18975, - "39": 0.18731, - "40": 0.18631, - "41": 0.18456, - "42": 0.18662, - "43": 0.18589, - "44": 0.18125, - "45": 0.19308, - "46": 0.17989, - "47": 0.1867, - "48": 0.18404, - "49": 0.18095, - "50": 0.18971, - "51": 0.25875, - "52": 0.20921, - "53": 0.19814, - "54": 0.19506, - "55": 0.17801, - "56": 0.17839, - "57": 0.18558, - "58": 0.1838, - "59": 0.17996, - "60": 0.18455, - "61": 0.19164, - "62": 0.1906, - "63": 0.17811, - "64": 0.18037, - "65": 0.18733, - "66": 0.18942, - "67": 0.18393, - "68": 0.18292, - "69": 0.18248, - "70": 0.1847, - "71": 0.17859, - "72": 0.18218, - "73": 0.18167, - "74": 0.177, - "75": 0.18191, - "76": 0.18191, - "77": 0.18558, - "78": 0.17902, - "79": 0.18474, - "80": 0.18875, - "81": 0.19061, - "82": 0.20019, - "83": 0.1804, - "84": 0.19066, - "85": 0.18001, - "86": 0.18562, - "87": 0.1847, - "88": 0.18147, - "89": 0.19573, - "90": 0.17983, - "91": 0.18358, - "92": 0.17809, - "93": 0.17549, - "94": 0.18735, - "95": 0.18122, - "96": 0.18269, - "97": 0.18541, - "98": 0.1788, - "99": 0.18398, - "100": 0.19234 + "2": 7.24019, + "3": 0.23582, + "4": 0.2146, + "5": 0.22942, + "6": 0.21484, + "7": 0.21536, + "8": 0.22845, + "9": 0.21127, + "10": 0.21126, + "11": 0.213, + "12": 0.20896, + "13": 0.21279, + "14": 0.20309, + "15": 0.21429, + "16": 0.19442, + "17": 0.20188, + "18": 0.20506, + "19": 0.20244, + "20": 0.20349, + "21": 0.19509, + "22": 0.20011, + "23": 0.19574, + "24": 0.19884, + "25": 0.20575, + "26": 0.19346, + "27": 0.18966, + "28": 0.20276, + "29": 0.20359, + "30": 0.20204, + "31": 0.19889, + "32": 0.20168, + "33": 0.20239, + "34": 0.19255, + "35": 0.19644, + "36": 0.18899, + "37": 0.19166, + "38": 0.19653, + "39": 0.1882, + "40": 0.19935, + "41": 0.1924, + "42": 0.19212, + "43": 0.19745, + "44": 0.1908, + "45": 0.20082, + "46": 0.18806, + "47": 0.19186, + "48": 0.19998, + "49": 0.19241, + "50": 0.18985, + "51": 0.25778, + "52": 0.2161, + "53": 0.19159, + "54": 0.19056, + "55": 0.18596, + "56": 0.18386, + "57": 0.19519, + "58": 0.19298, + "59": 0.1916, + "60": 0.18867, + "61": 0.19145, + "62": 0.19734, + "63": 0.18907, + "64": 0.18626, + "65": 0.18071, + "66": 0.18447, + "67": 0.18932, + "68": 0.18954, + "69": 0.19211, + "70": 0.18494, + "71": 0.18733, + "72": 0.18788, + "73": 0.19702, + "74": 0.19368, + "75": 0.19226, + "76": 0.18585, + "77": 0.20408, + "78": 0.19362, + "79": 0.18545, + "80": 0.18915, + "81": 0.20149, + "82": 0.18881, + "83": 0.18833, + "84": 0.19607, + "85": 0.18814, + "86": 0.19212, + "87": 0.19213, + "88": 0.18924, + "89": 0.18653, + "90": 0.18266, + "91": 0.19803, + "92": 0.18712, + "93": 0.18152, + "94": 0.19216, + "95": 0.18948, + "96": 0.18961, + "97": 0.19462, + "98": 0.19427, + "99": 0.1846, + "100": 0.19363 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json index bac266b9eef..c647bd3eeed 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.9252, - "4": 10.91732, - "5": 10.92716, - "6": 10.92029, - "7": 10.91669, - "8": 10.91796, - "9": 10.93887, - "10": 10.92403, - "11": 10.91598, + "1": 10.93064, + "2": 10.92264, + "3": 10.92451, + "4": 10.91716, + "5": 10.92714, + "6": 10.92064, + "7": 10.91647, + "8": 10.91788, + "9": 10.93809, + "10": 10.924, + "11": 10.91584, "12": 10.92558, - "13": 10.93454, - "14": 10.9158, - "15": 10.92838, - "16": 10.91826, - "17": 10.9274, - "18": 10.91269, - "19": 10.93069, - "20": 10.90057, - "21": 10.93975, - "22": 10.92531, - "23": 10.92976, - "24": 10.90567, - "25": 10.91494, - "26": 10.92515, - "27": 10.92786, - "28": 10.92017, - "29": 10.92534, - "30": 10.91409, - "31": 10.91023, - "32": 10.91855, - "33": 10.92026, - "34": 10.90054, - "35": 10.91897, - "36": 10.90741, - "37": 10.91267, - "38": 10.92379, - "39": 10.91251, - "40": 10.91043, - "41": 10.91309, - "42": 10.90891, - "43": 10.90503, - "44": 10.90748, - "45": 10.89756, - "46": 10.90681, - "47": 10.90215, - "48": 10.88414, - "49": 10.90022, - "50": 10.89863, - "51": 10.89746, - "52": 10.90395, - "53": 10.90118, - "54": 10.88728, - "55": 10.8869, - "56": 10.88881, - "57": 10.88731, - "58": 10.8888, - "59": 10.88231, - "60": 10.87378, - "61": 10.8731, - "62": 10.8684, - "63": 10.87569, - "64": 10.85934, - "65": 10.86089, - "66": 10.85671, - "67": 10.86152, - "68": 10.86019, - "69": 10.85018, - "70": 10.86781, - "71": 10.84867, - "72": 10.8405, - "73": 10.848, - "74": 10.83857, - "75": 10.83617, - "76": 10.83457, - "77": 10.82953, - "78": 10.82324, - "79": 10.82899, - "80": 10.82879, - "81": 10.81798, - "82": 10.81714, - "83": 10.80413, - "84": 10.78245, - "85": 10.78302, - "86": 10.80172, - "87": 10.79579, - "88": 10.79542, - "89": 10.77659, - "90": 10.78278, - "91": 10.79029, - "92": 10.78291, - "93": 10.75496, - "94": 10.76061, - "95": 10.77271, - "96": 10.73624, - "97": 10.74137, - "98": 10.74745, - "99": 10.76246, - "100": 10.74181 + "13": 10.9341, + "14": 10.91552, + "15": 10.92865, + "16": 10.91861, + "17": 10.92663, + "18": 10.91342, + "19": 10.93055, + "20": 10.89988, + "21": 10.94059, + "22": 10.92549, + "23": 10.92966, + "24": 10.90596, + "25": 10.91497, + "26": 10.92625, + "27": 10.92807, + "28": 10.92038, + "29": 10.92592, + "30": 10.91434, + "31": 10.91031, + "32": 10.919, + "33": 10.92014, + "34": 10.89998, + "35": 10.91944, + "36": 10.90693, + "37": 10.91287, + "38": 10.92352, + "39": 10.91196, + "40": 10.91094, + "41": 10.91323, + "42": 10.90919, + "43": 10.90589, + "44": 10.90761, + "45": 10.89741, + "46": 10.90636, + "47": 10.90203, + "48": 10.88495, + "49": 10.90015, + "50": 10.89905, + "51": 10.89814, + "52": 10.90393, + "53": 10.90158, + "54": 10.88808, + "55": 10.88643, + "56": 10.88868, + "57": 10.88748, + "58": 10.88859, + "59": 10.88263, + "60": 10.87391, + "61": 10.87367, + "62": 10.86781, + "63": 10.87607, + "64": 10.85882, + "65": 10.86026, + "66": 10.85712, + "67": 10.86113, + "68": 10.86032, + "69": 10.84979, + "70": 10.86738, + "71": 10.8492, + "72": 10.84019, + "73": 10.84868, + "74": 10.83869, + "75": 10.83621, + "76": 10.83434, + "77": 10.82938, + "78": 10.82319, + "79": 10.82846, + "80": 10.82987, + "81": 10.81761, + "82": 10.81704, + "83": 10.80437, + "84": 10.78281, + "85": 10.78319, + "86": 10.80029, + "87": 10.7956, + "88": 10.79539, + "89": 10.77706, + "90": 10.78274, + "91": 10.79015, + "92": 10.78298, + "93": 10.75507, + "94": 10.7607, + "95": 10.77273, + "96": 10.7355, + "97": 10.7421, + "98": 10.74705, + "99": 10.76264, + "100": 10.74262 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 49241.0, - "4": 48921.0, - "5": 50418.0, - "6": 49904.0, - "7": 51727.0, - "8": 49428.0, - "9": 50948.0, - "10": 52087.0, - "11": 50232.0, - "12": 48851.0, - "13": 52086.0, - "14": 50015.0, - "15": 47551.0, - "16": 49640.0, - "17": 49732.0, - "18": 51675.0, - "19": 50978.0, - "20": 52800.0, - "21": 49547.0, - "22": 49732.0, - "23": 53396.0, - "24": 51250.0, - "25": 48900.0, - "26": 52786.0, - "27": 51903.0, - "28": 48333.0, - "29": 49997.0, - "30": 51256.0, - "31": 52920.0, - "32": 49183.0, - "33": 52496.0, - "34": 50444.0, - "35": 50976.0, - "36": 50903.0, - "37": 49604.0, - "38": 51329.0, - "39": 50700.0, - "40": 50997.0, - "41": 48307.0, - "42": 50284.0, - "43": 51039.0, - "44": 47888.0, - "45": 54059.0, - "46": 47726.0, - "47": 50619.0, - "48": 50261.0, - "49": 55116.0, - "50": 50618.0, - "51": 49136.0, - "52": 50826.0, - "53": 49526.0, - "54": 49051.0, - "55": 48764.0, - "56": 48419.0, - "57": 48283.0, - "58": 52488.0, - "59": 50348.0, - "60": 49754.0, - "61": 47059.0, - "62": 50826.0, - "63": 48461.0, - "64": 55874.0, - "65": 51256.0, - "66": 48951.0, - "67": 51893.0, - "68": 49259.0, - "69": 54945.0, - "70": 50024.0, - "71": 48846.0, - "72": 51735.0, - "73": 52873.0, - "74": 49777.0, - "75": 50232.0, - "76": 51309.0, - "77": 50257.0, - "78": 49979.0, - "79": 48610.0, - "80": 51996.0, - "81": 50042.0, - "82": 47216.0, - "83": 52389.0, - "84": 48624.0, - "85": 50422.0, - "86": 49534.0, - "87": 46782.0, - "88": 48381.0, - "89": 49533.0, - "90": 52461.0, - "91": 50240.0, - "92": 50070.0, - "93": 50832.0, - "94": 51690.0, - "95": 48408.0, - "96": 49903.0, - "97": 49691.0, - "98": 48279.0, - "99": 48860.0, - "100": 47315.0 + "1": 49180.0, + "2": 50476.0, + "3": 49283.0, + "4": 48797.0, + "5": 50222.0, + "6": 50071.0, + "7": 51516.0, + "8": 49687.0, + "9": 50960.0, + "10": 52404.0, + "11": 50454.0, + "12": 48350.0, + "13": 52171.0, + "14": 49951.0, + "15": 48080.0, + "16": 49110.0, + "17": 50288.0, + "18": 52118.0, + "19": 51099.0, + "20": 52646.0, + "21": 49816.0, + "22": 49955.0, + "23": 53512.0, + "24": 51380.0, + "25": 48547.0, + "26": 52530.0, + "27": 51993.0, + "28": 48540.0, + "29": 50032.0, + "30": 51384.0, + "31": 53098.0, + "32": 49324.0, + "33": 51925.0, + "34": 50079.0, + "35": 51371.0, + "36": 51207.0, + "37": 49665.0, + "38": 50936.0, + "39": 50632.0, + "40": 50968.0, + "41": 48043.0, + "42": 50241.0, + "43": 50548.0, + "44": 47518.0, + "45": 54359.0, + "46": 47844.0, + "47": 50697.0, + "48": 50607.0, + "49": 54927.0, + "50": 51129.0, + "51": 48773.0, + "52": 50817.0, + "53": 49252.0, + "54": 48794.0, + "55": 49077.0, + "56": 48641.0, + "57": 48141.0, + "58": 52709.0, + "59": 50075.0, + "60": 49760.0, + "61": 47761.0, + "62": 50557.0, + "63": 48459.0, + "64": 56353.0, + "65": 51179.0, + "66": 48545.0, + "67": 51570.0, + "68": 49653.0, + "69": 54155.0, + "70": 49808.0, + "71": 49534.0, + "72": 52109.0, + "73": 52558.0, + "74": 49904.0, + "75": 50500.0, + "76": 52020.0, + "77": 49771.0, + "78": 49836.0, + "79": 48741.0, + "80": 52069.0, + "81": 50079.0, + "82": 46782.0, + "83": 52982.0, + "84": 48744.0, + "85": 50660.0, + "86": 50050.0, + "87": 46875.0, + "88": 48017.0, + "89": 48621.0, + "90": 51995.0, + "91": 50317.0, + "92": 50374.0, + "93": 50702.0, + "94": 51589.0, + "95": 48646.0, + "96": 50031.0, + "97": 49732.0, + "98": 48573.0, + "99": 48599.0, + "100": 47055.0 } }, "mem-allocated-bytes": { @@ -218,105 +218,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 870734336.0, - "2": 870755840.0, + "1": 870735872.0, + "2": 870754816.0, "3": 870738944.0, - "4": 870736384.0, - "5": 870749184.0, - "6": 870733824.0, - "7": 870728704.0, - "8": 870757376.0, - "9": 870707712.0, - "10": 870706176.0, - "11": 870695424.0, - "12": 870721536.0, - "13": 870721536.0, - "14": 870710272.0, - "15": 870740992.0, - "16": 870730240.0, - "17": 870743040.0, - "18": 870720000.0, - "19": 870715392.0, - "20": 870729216.0, - "21": 870750720.0, - "22": 870684672.0, - "23": 870728192.0, + "4": 870736896.0, + "5": 870747136.0, + "6": 870734848.0, + "7": 870729728.0, + "8": 870758400.0, + "9": 870706688.0, + "10": 870708224.0, + "11": 870691840.0, + "12": 870722560.0, + "13": 870722048.0, + "14": 870712320.0, + "15": 870742016.0, + "16": 870731264.0, + "17": 870742528.0, + "18": 870718976.0, + "19": 870715904.0, + "20": 870726656.0, + "21": 870749696.0, + "22": 870685696.0, + "23": 870727168.0, "24": 870740992.0, - "25": 870703616.0, + "25": 870705664.0, "26": 870743040.0, - "27": 870746112.0, + "27": 870745600.0, "28": 870710272.0, - "29": 870728192.0, - "30": 870737920.0, - "31": 870701568.0, - "32": 870721536.0, - "33": 870708736.0, + "29": 870727680.0, + "30": 870737408.0, + "31": 870700544.0, + "32": 870722048.0, + "33": 870707712.0, "34": 870750208.0, - "35": 870679552.0, - "36": 870751744.0, + "35": 870678016.0, + "36": 870753792.0, "37": 870719488.0, - "38": 870747648.0, - "39": 870707712.0, + "38": 870744064.0, + "39": 870709760.0, "40": 870759424.0, - "41": 870723584.0, - "42": 870704128.0, - "43": 870708736.0, - "44": 870715392.0, + "41": 870726656.0, + "42": 870705664.0, + "43": 870706176.0, + "44": 870716416.0, "45": 870730752.0, - "46": 870681088.0, - "47": 870702592.0, - "48": 870694400.0, - "49": 870727680.0, - "50": 870706176.0, - "51": 870675968.0, - "52": 870722560.0, + "46": 870681600.0, + "47": 870704640.0, + "48": 870693888.0, + "49": 870726144.0, + "50": 870706688.0, + "51": 870676480.0, + "52": 870724096.0, "53": 870680576.0, - "54": 870742528.0, - "55": 870687232.0, - "56": 870735872.0, - "57": 870709248.0, - "58": 870713344.0, + "54": 870741504.0, + "55": 870686208.0, + "56": 870737920.0, + "57": 870706176.0, + "58": 870713856.0, "59": 870715904.0, - "60": 870678528.0, - "61": 870660608.0, - "62": 870694912.0, - "63": 870702592.0, - "64": 870717952.0, - "65": 870720000.0, - "66": 870642176.0, - "67": 870725632.0, - "68": 870674944.0, - "69": 870699008.0, - "70": 870664704.0, - "71": 870688768.0, - "72": 870688256.0, - "73": 870685184.0, - "74": 870651904.0, - "75": 870676480.0, - "76": 870678528.0, - "77": 870715392.0, - "78": 870711296.0, - "79": 870681088.0, - "80": 870627328.0, - "81": 870669824.0, - "82": 870642688.0, + "60": 870680576.0, + "61": 870662656.0, + "62": 870695936.0, + "63": 870699008.0, + "64": 870713344.0, + "65": 870717952.0, + "66": 870641152.0, + "67": 870729216.0, + "68": 870673920.0, + "69": 870702080.0, + "70": 870668288.0, + "71": 870687744.0, + "72": 870685184.0, + "73": 870686208.0, + "74": 870650368.0, + "75": 870676992.0, + "76": 870678016.0, + "77": 870716928.0, + "78": 870710784.0, + "79": 870678528.0, + "80": 870626816.0, + "81": 870669312.0, + "82": 870644736.0, "83": 870636032.0, - "84": 870680576.0, + "84": 870677504.0, "85": 870650880.0, - "86": 870646784.0, - "87": 870637056.0, - "88": 870647808.0, - "89": 870635520.0, - "90": 870673920.0, - "91": 870638592.0, - "92": 870637056.0, + "86": 870644736.0, + "87": 870636032.0, + "88": 870648832.0, + "89": 870634496.0, + "90": 870674432.0, + "91": 870637056.0, + "92": 870639104.0, "93": 870653952.0, - "94": 870636032.0, - "95": 870631936.0, - "96": 870638080.0, - "97": 870631936.0, + "94": 870635520.0, + "95": 870631424.0, + "96": 870637056.0, + "97": 870630912.0, "98": 870680064.0, - "99": 870618112.0, + "99": 870621184.0, "100": 870622720.0 } }, @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3339315712.0, - "2": 3489557504.0, - "3": 3489557504.0, - "4": 3489557504.0, - "5": 3489557504.0, - "6": 3489557504.0, - "7": 3489557504.0, - "8": 3489557504.0, - "9": 3489557504.0, - "10": 3489557504.0, - "11": 3489557504.0, - "12": 3489557504.0, - "13": 3489557504.0, - "14": 3489610752.0, - "15": 3489610752.0, - "16": 3489610752.0, - "17": 3489610752.0, - "18": 3489610752.0, - "19": 3489610752.0, - "20": 3489610752.0, - "21": 3489610752.0, - "22": 3489610752.0, - "23": 3489610752.0, - "24": 3489610752.0, - "25": 3489610752.0, - "26": 3489610752.0, - "27": 3489610752.0, - "28": 3489610752.0, - "29": 3489610752.0, - "30": 3489610752.0, - "31": 3489610752.0, - "32": 3489610752.0, - "33": 3489610752.0, - "34": 3489610752.0, - "35": 3489610752.0, - "36": 3489610752.0, - "37": 3489610752.0, - "38": 3489610752.0, - "39": 3489610752.0, - "40": 3489610752.0, - "41": 3489610752.0, - "42": 3489610752.0, - "43": 3489610752.0, - "44": 3489610752.0, - "45": 3489610752.0, - "46": 3489610752.0, - "47": 3489610752.0, - "48": 3489610752.0, - "49": 3489610752.0, - "50": 3489610752.0, - "51": 3489610752.0, - "52": 3489610752.0, - "53": 3489610752.0, - "54": 3489610752.0, - "55": 3489610752.0, - "56": 3489610752.0, - "57": 3489610752.0, - "58": 3489610752.0, - "59": 3489610752.0, - "60": 3489610752.0, - "61": 3489610752.0, - "62": 3489610752.0, - "63": 3489610752.0, - "64": 3489610752.0, - "65": 3489610752.0, - "66": 3489610752.0, - "67": 3489610752.0, - "68": 3489610752.0, - "69": 3489610752.0, - "70": 3489610752.0, - "71": 3489610752.0, - "72": 3489610752.0, - "73": 3489610752.0, - "74": 3489610752.0, - "75": 3489610752.0, - "76": 3489610752.0, - "77": 3489610752.0, - "78": 3489610752.0, - "79": 3489610752.0, - "80": 3489610752.0, - "81": 3489610752.0, - "82": 3489610752.0, - "83": 3489610752.0, - "84": 3489610752.0, - "85": 3489610752.0, - "86": 3489610752.0, - "87": 3489610752.0, - "88": 3489610752.0, - "89": 3489610752.0, - "90": 3489610752.0, - "91": 3489610752.0, - "92": 3489610752.0, - "93": 3489610752.0, - "94": 3489610752.0, - "95": 3489610752.0, - "96": 3489610752.0, - "97": 3489610752.0, - "98": 3489610752.0, - "99": 3489610752.0, - "100": 3489610752.0 + "1": 3338426368.0, + "2": 3490031616.0, + "3": 3490031616.0, + "4": 3490031616.0, + "5": 3490031616.0, + "6": 3490031616.0, + "7": 3490031616.0, + "8": 3490031616.0, + "9": 3490031616.0, + "10": 3490031616.0, + "11": 3490031616.0, + "12": 3490031616.0, + "13": 3490031616.0, + "14": 3490031616.0, + "15": 3490031616.0, + "16": 3490031616.0, + "17": 3490031616.0, + "18": 3490031616.0, + "19": 3490031616.0, + "20": 3490031616.0, + "21": 3490031616.0, + "22": 3490031616.0, + "23": 3490031616.0, + "24": 3490031616.0, + "25": 3490031616.0, + "26": 3490031616.0, + "27": 3490031616.0, + "28": 3490031616.0, + "29": 3490031616.0, + "30": 3490031616.0, + "31": 3490031616.0, + "32": 3490031616.0, + "33": 3490031616.0, + "34": 3490031616.0, + "35": 3490031616.0, + "36": 3490031616.0, + "37": 3490031616.0, + "38": 3490031616.0, + "39": 3490031616.0, + "40": 3490031616.0, + "41": 3490031616.0, + "42": 3490031616.0, + "43": 3490031616.0, + "44": 3490031616.0, + "45": 3490031616.0, + "46": 3490031616.0, + "47": 3490031616.0, + "48": 3490031616.0, + "49": 3490031616.0, + "50": 3490031616.0, + "51": 3490031616.0, + "52": 3490031616.0, + "53": 3490031616.0, + "54": 3490031616.0, + "55": 3490031616.0, + "56": 3490031616.0, + "57": 3490031616.0, + "58": 3490031616.0, + "59": 3490031616.0, + "60": 3490031616.0, + "61": 3490031616.0, + "62": 3490031616.0, + "63": 3490031616.0, + "64": 3490031616.0, + "65": 3490031616.0, + "66": 3490031616.0, + "67": 3490031616.0, + "68": 3490031616.0, + "69": 3490031616.0, + "70": 3490031616.0, + "71": 3490031616.0, + "72": 3490031616.0, + "73": 3490031616.0, + "74": 3490031616.0, + "75": 3490031616.0, + "76": 3490031616.0, + "77": 3490031616.0, + "78": 3490031616.0, + "79": 3490031616.0, + "80": 3490031616.0, + "81": 3490031616.0, + "82": 3490031616.0, + "83": 3490031616.0, + "84": 3490031616.0, + "85": 3490031616.0, + "86": 3490031616.0, + "87": 3490031616.0, + "88": 3490031616.0, + "89": 3490031616.0, + "90": 3490031616.0, + "91": 3490031616.0, + "92": 3490031616.0, + "93": 3490031616.0, + "94": 3490031616.0, + "95": 3490031616.0, + "96": 3490031616.0, + "97": 3490031616.0, + "98": 3490031616.0, + "99": 3490031616.0, + "100": 3490031616.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.58549, - "3": 0.45873, - "4": 0.72389, - "5": 0.43572, - "6": 0.61689, - "7": 0.71091, - "8": 0.80567, - "9": 0.53532, - "10": 0.7268, - "11": 1.09113, - "12": 0.41894, - "13": 0.61108, - "14": 0.96012, - "15": 0.42046, - "16": 1.19937, - "17": 0.68288, - "18": 1.00063, - "19": 0.42525, - "20": 0.43066, - "21": 0.90114, - "22": 0.78844, - "23": 1.14833, - "24": 0.41739, - "25": 0.50646, - "26": 0.94527, - "27": 0.42032, - "28": 0.58231, - "29": 0.88167, - "30": 0.42641, - "31": 0.41707, - "32": 0.86526, - "33": 0.54382, - "34": 1.09909, - "35": 0.40999, - "36": 0.62898, - "37": 0.56949, - "38": 0.46383, - "39": 0.62988, - "40": 0.80889, - "41": 0.69177, - "42": 0.85919, - "43": 0.54109, - "44": 0.94286, - "45": 0.52102, - "46": 0.81405, - "47": 0.68935, - "48": 1.18723, - "49": 0.46696, - "50": 0.45085, - "51": 0.48617, - "52": 0.46498, - "53": 0.43002, - "54": 0.78754, - "55": 0.77661, - "56": 0.91987, - "57": 0.43668, - "58": 0.57538, - "59": 0.8118, - "60": 0.64391, - "61": 0.42024, - "62": 0.61178, - "63": 0.59716, - "64": 0.85997, - "65": 0.51173, - "66": 0.75369, - "67": 0.60196, - "68": 0.41303, - "69": 0.51359, - "70": 1.07059, - "71": 0.40741, - "72": 0.42067, - "73": 0.57342, - "74": 0.47313, - "75": 1.01083, - "76": 0.43509, - "77": 0.43961, - "78": 1.41587, - "79": 0.40922, - "80": 0.64319, - "81": 0.71607, - "82": 0.58617, - "83": 0.81488, - "84": 0.68728, - "85": 0.41029, - "86": 0.41426, - "87": 0.68579, - "88": 0.87244, - "89": 0.45803, - "90": 0.52494, - "91": 0.50687, - "92": 0.9093, - "93": 0.44208, - "94": 0.41724, - "95": 0.74468, - "96": 0.48072, - "97": 0.53053, - "98": 0.72816, - "99": 1.40355, - "100": 0.4221 + "2": 5.83302, + "3": 0.39835, + "4": 0.38075, + "5": 0.37076, + "6": 0.4933, + "7": 0.74612, + "8": 0.78858, + "9": 0.57808, + "10": 0.35052, + "11": 1.14199, + "12": 0.35131, + "13": 0.43605, + "14": 1.01433, + "15": 0.34635, + "16": 0.8652, + "17": 0.4883, + "18": 0.78111, + "19": 0.34941, + "20": 0.34539, + "21": 0.80911, + "22": 0.63224, + "23": 0.59018, + "24": 0.35026, + "25": 0.48482, + "26": 0.69507, + "27": 0.35024, + "28": 0.62934, + "29": 0.63054, + "30": 0.35463, + "31": 0.36418, + "32": 0.92689, + "33": 0.55794, + "34": 0.61214, + "35": 0.34834, + "36": 0.48957, + "37": 0.54155, + "38": 0.37385, + "39": 0.57673, + "40": 0.61101, + "41": 0.56321, + "42": 0.72541, + "43": 0.37383, + "44": 0.87492, + "45": 0.53872, + "46": 0.61778, + "47": 0.55835, + "48": 0.69866, + "49": 0.35686, + "50": 0.63177, + "51": 0.41131, + "52": 0.42473, + "53": 0.35518, + "54": 0.72158, + "55": 0.58026, + "56": 0.75569, + "57": 0.4147, + "58": 0.35309, + "59": 0.78616, + "60": 0.66808, + "61": 0.35136, + "62": 0.563, + "63": 0.48971, + "64": 0.75453, + "65": 0.48762, + "66": 0.71564, + "67": 0.58774, + "68": 0.46593, + "69": 0.47007, + "70": 0.96668, + "71": 0.34914, + "72": 0.50835, + "73": 0.56066, + "74": 0.41042, + "75": 0.93675, + "76": 0.39087, + "77": 0.41741, + "78": 1.30294, + "79": 0.3568, + "80": 0.61012, + "81": 0.63075, + "82": 0.50353, + "83": 0.67509, + "84": 0.65086, + "85": 0.35856, + "86": 0.35432, + "87": 0.78386, + "88": 0.90597, + "89": 0.54771, + "90": 0.48869, + "91": 0.51287, + "92": 0.6772, + "93": 0.35448, + "94": 0.5901, + "95": 0.68505, + "96": 0.36981, + "97": 0.61461, + "98": 0.6597, + "99": 1.25701, + "100": 0.35321 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index b53c321d4db..11c3cb6c672 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94254, - "2": 10.91055, - "3": 10.92694, - "4": 10.91735, - "5": 10.92135, - "6": 10.92675, - "7": 10.92142, - "8": 10.91839, - "9": 10.92956, - "10": 10.92369, - "11": 10.90819, - "12": 10.92314, - "13": 10.90646, - "14": 10.89552, - "15": 10.88233, - "16": 10.87919, - "17": 10.89075, - "18": 10.86032, - "19": 10.87423, - "20": 10.80771, - "21": 10.79658, - "22": 10.78022, - "23": 10.78113, - "24": 10.73807, - "25": 10.73956, - "26": 10.72339, - "27": 10.71536, - "28": 10.65034, - "29": 10.61992, - "30": 10.5876, - "31": 10.58328, - "32": 10.57421, - "33": 10.54175, - "34": 10.50515, - "35": 10.52189, - "36": 10.48057, - "37": 10.44356, - "38": 10.4595, - "39": 10.42486, - "40": 10.41442, - "41": 10.3917, - "42": 10.38806, - "43": 10.34951, - "44": 10.34003, - "45": 10.34165, - "46": 10.30975, - "47": 10.28885, - "48": 10.2537, - "49": 10.24213, - "50": 10.24933, - "51": 10.24422, - "52": 10.19431, - "53": 10.19081, - "54": 10.16475, - "55": 10.13357, - "56": 10.16655, - "57": 10.13693, - "58": 10.15647, - "59": 10.10405, - "60": 10.10771, - "61": 10.0758, - "62": 10.03959, - "63": 10.11148, - "64": 10.05923, - "65": 10.03145, - "66": 10.05788, - "67": 10.03114, - "68": 9.98969, - "69": 10.00266, - "70": 9.99298, - "71": 10.02384, - "72": 9.98601, - "73": 9.97321, + "1": 10.94223, + "2": 10.91037, + "3": 10.92695, + "4": 10.91767, + "5": 10.92123, + "6": 10.92664, + "7": 10.9217, + "8": 10.91836, + "9": 10.92965, + "10": 10.92315, + "11": 10.90829, + "12": 10.92307, + "13": 10.90609, + "14": 10.89523, + "15": 10.88222, + "16": 10.87896, + "17": 10.89049, + "18": 10.86024, + "19": 10.8748, + "20": 10.80806, + "21": 10.79625, + "22": 10.77972, + "23": 10.7812, + "24": 10.73749, + "25": 10.73943, + "26": 10.72335, + "27": 10.71554, + "28": 10.65056, + "29": 10.62001, + "30": 10.58815, + "31": 10.5838, + "32": 10.57437, + "33": 10.54146, + "34": 10.5048, + "35": 10.52203, + "36": 10.48035, + "37": 10.4436, + "38": 10.45979, + "39": 10.42469, + "40": 10.41474, + "41": 10.39207, + "42": 10.38861, + "43": 10.34961, + "44": 10.33972, + "45": 10.34145, + "46": 10.30945, + "47": 10.28903, + "48": 10.25379, + "49": 10.24217, + "50": 10.24935, + "51": 10.24386, + "52": 10.19468, + "53": 10.19068, + "54": 10.1649, + "55": 10.13346, + "56": 10.16662, + "57": 10.1368, + "58": 10.15637, + "59": 10.10414, + "60": 10.10781, + "61": 10.07602, + "62": 10.03947, + "63": 10.11164, + "64": 10.05938, + "65": 10.03151, + "66": 10.05805, + "67": 10.03102, + "68": 9.98965, + "69": 10.00291, + "70": 9.99282, + "71": 10.02377, + "72": 9.98607, + "73": 9.97326, "74": 9.96181, - "75": 9.92935, - "76": 9.97534, - "77": 9.96643, - "78": 9.90591, - "79": 9.91751, - "80": 9.92945, - "81": 9.95014, - "82": 9.90413, - "83": 9.86106, - "84": 9.79605, - "85": 9.78046, - "86": 9.89395, - "87": 9.92023, - "88": 9.89599, - "89": 9.83296, - "90": 9.82062, - "91": 9.83514, - "92": 9.82674, - "93": 9.76793, - "94": 9.84068, - "95": 9.83841, - "96": 9.8204, - "97": 9.75773, - "98": 9.79348, - "99": 9.83553, - "100": 9.73545 + "75": 9.92948, + "76": 9.97549, + "77": 9.96656, + "78": 9.90586, + "79": 9.91757, + "80": 9.92926, + "81": 9.95031, + "82": 9.90425, + "83": 9.86098, + "84": 9.79635, + "85": 9.78073, + "86": 9.89398, + "87": 9.92005, + "88": 9.89597, + "89": 9.83309, + "90": 9.8208, + "91": 9.83504, + "92": 9.8266, + "93": 9.76812, + "94": 9.84073, + "95": 9.83814, + "96": 9.82054, + "97": 9.75795, + "98": 9.79373, + "99": 9.83563, + "100": 9.73571 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1411.0, - "2": 1137.0, - "3": 1232.0, - "4": 1244.0, - "5": 1217.0, - "6": 1268.0, - "7": 1535.0, - "8": 1336.0, - "9": 1216.0, - "10": 1248.0, - "11": 1205.0, - "12": 1348.0, - "13": 1369.0, - "14": 1526.0, - "15": 1242.0, - "16": 1252.0, - "17": 1215.0, - "18": 1243.0, - "19": 1287.0, - "20": 1327.0, - "21": 1256.0, - "22": 1280.0, - "23": 1220.0, - "24": 1286.0, - "25": 1282.0, - "26": 1101.0, - "27": 1261.0, - "28": 1190.0, - "29": 1102.0, - "30": 1084.0, - "31": 1148.0, - "32": 1158.0, - "33": 1171.0, - "34": 1206.0, - "35": 1270.0, - "36": 1261.0, - "37": 1171.0, - "38": 1085.0, - "39": 1091.0, - "40": 1185.0, - "41": 1215.0, - "42": 1099.0, - "43": 1165.0, - "44": 48220.0, - "45": 1319.0, - "46": 1180.0, - "47": 2395.0, - "48": 8402.0, - "49": 2284.0, - "50": 1243.0, - "51": 5318.0, - "52": 1050439.0, - "53": 1115984.0, - "54": 13572.0, - "55": 1104619.0, - "56": 51521.0, - "57": 1050311.0, - "58": 120308.0, - "59": 1050412.0, - "60": 1064735.0, - "61": 1100608.0, - "62": 1050496.0, - "63": 139692.0, - "64": 2103808.0, - "65": 2164916.0, - "66": 45425.0, - "67": 1117177.0, - "68": 1121286.0, - "69": 2107748.0, - "70": 2142661.0, - "71": 2149785.0, - "72": 2203058.0, - "73": 2099685.0, - "74": 1104890.0, - "75": 2155870.0, - "76": 2181677.0, - "77": 1148035.0, - "78": 2103861.0, - "79": 2145742.0, - "80": 1102831.0, - "81": 1124411.0, - "82": 3207346.0, - "83": 57851.0, - "84": 100811.0, - "85": 1092414.0, - "86": 2100662.0, - "87": 3190894.0, - "88": 1101861.0, - "89": 3148799.0, - "90": 3206244.0, - "91": 3151733.0, - "92": 3148823.0, - "93": 1102810.0, - "94": 3204059.0, - "95": 3149017.0, - "96": 2207416.0, - "97": 3154036.0, - "98": 3204428.0, - "99": 2171535.0, - "100": 3199111.0 + "1": 1356.0, + "2": 1146.0, + "3": 1308.0, + "4": 1201.0, + "5": 1181.0, + "6": 1356.0, + "7": 1661.0, + "8": 1309.0, + "9": 1246.0, + "10": 1257.0, + "11": 1204.0, + "12": 1346.0, + "13": 1385.0, + "14": 1393.0, + "15": 1241.0, + "16": 1189.0, + "17": 1180.0, + "18": 1293.0, + "19": 1321.0, + "20": 1383.0, + "21": 1333.0, + "22": 1235.0, + "23": 1212.0, + "24": 1255.0, + "25": 1268.0, + "26": 1121.0, + "27": 1302.0, + "28": 1254.0, + "29": 1139.0, + "30": 1069.0, + "31": 1202.0, + "32": 1194.0, + "33": 1138.0, + "34": 1179.0, + "35": 1157.0, + "36": 1262.0, + "37": 1041.0, + "38": 1010.0, + "39": 1126.0, + "40": 1282.0, + "41": 1192.0, + "42": 1131.0, + "43": 1163.0, + "44": 47171.0, + "45": 1307.0, + "46": 1144.0, + "47": 2329.0, + "48": 1185.0, + "49": 2275.0, + "50": 1311.0, + "51": 50408.0, + "52": 1050367.0, + "53": 1115953.0, + "54": 13597.0, + "55": 1104617.0, + "56": 52522.0, + "57": 1050274.0, + "58": 120266.0, + "59": 1055549.0, + "60": 1064790.0, + "61": 98631.0, + "62": 1050443.0, + "63": 140751.0, + "64": 2103804.0, + "65": 2164940.0, + "66": 45473.0, + "67": 1117211.0, + "68": 1163244.0, + "69": 2145629.0, + "70": 2142654.0, + "71": 2197856.0, + "72": 2113996.0, + "73": 2099710.0, + "74": 1111040.0, + "75": 2162030.0, + "76": 2180613.0, + "77": 1148043.0, + "78": 2103914.0, + "79": 2145703.0, + "80": 1109006.0, + "81": 1124413.0, + "82": 2209422.0, + "83": 57887.0, + "84": 1099708.0, + "85": 1143598.0, + "86": 2100695.0, + "87": 3190841.0, + "88": 106080.0, + "89": 3148860.0, + "90": 3206232.0, + "91": 3151748.0, + "92": 3148863.0, + "93": 1103759.0, + "94": 3203989.0, + "95": 3148998.0, + "96": 2207348.0, + "97": 3154093.0, + "98": 3204328.0, + "99": 2171573.0, + "100": 3198980.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 788192256.0, - "2": 788206592.0, - "3": 788197888.0, - "4": 788182016.0, - "5": 788190720.0, - "6": 788174848.0, - "7": 788195328.0, - "8": 788164608.0, - "9": 788191232.0, - "10": 788185088.0, - "11": 788160512.0, - "12": 788149248.0, - "13": 788237824.0, - "14": 788246528.0, - "15": 788176384.0, - "16": 788174848.0, - "17": 788187136.0, - "18": 788146176.0, - "19": 788199424.0, - "20": 788144128.0, - "21": 788177408.0, - "22": 788133376.0, - "23": 788098560.0, - "24": 788127744.0, - "25": 788103680.0, - "26": 788181504.0, - "27": 788113920.0, - "28": 788083200.0, - "29": 788133888.0, - "30": 788067840.0, - "31": 788068864.0, - "32": 788082688.0, - "33": 788052992.0, - "34": 788099072.0, - "35": 788082688.0, - "36": 788040192.0, - "37": 788071936.0, - "38": 788088832.0, - "39": 788029952.0, - "40": 788040192.0, - "41": 788059648.0, - "42": 788055040.0, - "43": 788051456.0, - "44": 788024832.0, - "45": 788004864.0, - "46": 788048384.0, - "47": 788029952.0, - "48": 788004864.0, - "49": 787987456.0, - "50": 787956224.0, - "51": 787942400.0, - "52": 787940864.0, - "53": 787940864.0, - "54": 787928064.0, - "55": 787914240.0, - "56": 787934720.0, - "57": 787944960.0, - "58": 787950592.0, - "59": 787915776.0, - "60": 787945984.0, - "61": 787897856.0, - "62": 787923456.0, - "63": 787927552.0, - "64": 787934208.0, - "65": 787929600.0, - "66": 787887616.0, - "67": 787930112.0, - "68": 787922432.0, - "69": 787922944.0, - "70": 787895296.0, - "71": 787890176.0, - "72": 787949056.0, - "73": 787913728.0, - "74": 787871744.0, - "75": 787888128.0, - "76": 787862528.0, - "77": 787886080.0, - "78": 787860480.0, - "79": 787842048.0, - "80": 787816448.0, - "81": 787819520.0, - "82": 787778048.0, - "83": 787818496.0, - "84": 787830272.0, - "85": 787828224.0, - "86": 787830784.0, - "87": 787715584.0, - "88": 787748864.0, - "89": 787815936.0, - "90": 787814400.0, - "91": 787840512.0, - "92": 787814400.0, - "93": 787805696.0, - "94": 787896832.0, - "95": 787808256.0, - "96": 787827712.0, - "97": 787846656.0, - "98": 787834880.0, - "99": 787803136.0, - "100": 787847680.0 + "1": 788639744.0, + "2": 788651008.0, + "3": 788643328.0, + "4": 788628480.0, + "5": 788636672.0, + "6": 788622336.0, + "7": 788641280.0, + "8": 788612608.0, + "9": 788637184.0, + "10": 788629504.0, + "11": 788607488.0, + "12": 788597248.0, + "13": 788684288.0, + "14": 788691456.0, + "15": 788621824.0, + "16": 788622336.0, + "17": 788631552.0, + "18": 788590592.0, + "19": 788645888.0, + "20": 788592640.0, + "21": 788622336.0, + "22": 788581888.0, + "23": 788545024.0, + "24": 788574208.0, + "25": 788550144.0, + "26": 788628992.0, + "27": 788562432.0, + "28": 788528640.0, + "29": 788578816.0, + "30": 788515840.0, + "31": 788516352.0, + "32": 788528128.0, + "33": 788499968.0, + "34": 788546560.0, + "35": 788528640.0, + "36": 788487680.0, + "37": 788518400.0, + "38": 788532224.0, + "39": 788476928.0, + "40": 788488192.0, + "41": 788506624.0, + "42": 788504064.0, + "43": 788497920.0, + "44": 788470784.0, + "45": 788451328.0, + "46": 788493824.0, + "47": 788476416.0, + "48": 788452864.0, + "49": 788434432.0, + "50": 788404224.0, + "51": 788392448.0, + "52": 788389376.0, + "53": 788388864.0, + "54": 788375552.0, + "55": 788362240.0, + "56": 788380672.0, + "57": 788392448.0, + "58": 788394496.0, + "59": 788360704.0, + "60": 788393984.0, + "61": 788348928.0, + "62": 788369920.0, + "63": 788376576.0, + "64": 788382720.0, + "65": 788378112.0, + "66": 788334592.0, + "67": 788378624.0, + "68": 788372480.0, + "69": 788373504.0, + "70": 788344320.0, + "71": 788338688.0, + "72": 788396032.0, + "73": 788360704.0, + "74": 788320256.0, + "75": 788337152.0, + "76": 788312064.0, + "77": 788334592.0, + "78": 788309504.0, + "79": 788289024.0, + "80": 788264448.0, + "81": 788267520.0, + "82": 788224000.0, + "83": 788265984.0, + "84": 788276224.0, + "85": 788275712.0, + "86": 788277248.0, + "87": 788162048.0, + "88": 788196352.0, + "89": 788263424.0, + "90": 788261376.0, + "91": 788287488.0, + "92": 788261376.0, + "93": 788253184.0, + "94": 788343808.0, + "95": 788255232.0, + "96": 788274688.0, + "97": 788295680.0, + "98": 788281856.0, + "99": 788251136.0, + "100": 788296192.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2944822784.0, - "2": 3126373888.0, - "3": 3126373888.0, - "4": 3126373888.0, - "5": 3126373888.0, - "6": 3126373888.0, - "7": 3126373888.0, - "8": 3126373888.0, - "9": 3126373888.0, - "10": 3126373888.0, - "11": 3126373888.0, - "12": 3126373888.0, - "13": 3147687936.0, - "14": 3152973312.0, - "15": 3152973312.0, - "16": 3152973312.0, - "17": 3152973312.0, - "18": 3152973312.0, - "19": 3152973312.0, - "20": 3152973312.0, - "21": 3152973312.0, - "22": 3152973312.0, - "23": 3152973312.0, - "24": 3152973312.0, - "25": 3152973312.0, - "26": 3152973312.0, - "27": 3152973312.0, - "28": 3152973312.0, - "29": 3152973312.0, - "30": 3152973312.0, - "31": 3152973312.0, - "32": 3152973312.0, - "33": 3152973312.0, - "34": 3152973312.0, - "35": 3152973312.0, - "36": 3152973312.0, - "37": 3152973312.0, - "38": 3152973312.0, - "39": 3152973312.0, - "40": 3152973312.0, - "41": 3152973312.0, - "42": 3152973312.0, - "43": 3152973312.0, - "44": 3152973312.0, - "45": 3152973312.0, - "46": 3152973312.0, - "47": 3152973312.0, - "48": 3152973312.0, - "49": 3152973312.0, - "50": 3152973312.0, - "51": 3152973312.0, - "52": 3152973312.0, - "53": 3152973312.0, - "54": 3152973312.0, - "55": 3152973312.0, - "56": 3152973312.0, - "57": 3152973312.0, - "58": 3152973312.0, - "59": 3152973312.0, - "60": 3152973312.0, - "61": 3152973312.0, - "62": 3152973312.0, - "63": 3152973312.0, - "64": 3152973312.0, - "65": 3152973312.0, - "66": 3152973312.0, - "67": 3152973312.0, - "68": 3152973312.0, - "69": 3152973312.0, - "70": 3152973312.0, - "71": 3152973312.0, - "72": 3152973312.0, - "73": 3152973312.0, - "74": 3152973312.0, - "75": 3152973312.0, - "76": 3152973312.0, - "77": 3152973312.0, - "78": 3152973312.0, - "79": 3152973312.0, - "80": 3152973312.0, - "81": 3152973312.0, - "82": 3152973312.0, - "83": 3152973312.0, - "84": 3152973312.0, - "85": 3152973312.0, - "86": 3152973312.0, - "87": 3152973312.0, - "88": 3152973312.0, - "89": 3152973312.0, - "90": 3152973312.0, - "91": 3152973312.0, - "92": 3152973312.0, - "93": 3152973312.0, - "94": 3152973312.0, - "95": 3152973312.0, - "96": 3152973312.0, - "97": 3152973312.0, - "98": 3152973312.0, - "99": 3152973312.0, - "100": 3152973312.0 + "1": 2942716416.0, + "2": 3126817792.0, + "3": 3126817792.0, + "4": 3126817792.0, + "5": 3126817792.0, + "6": 3126817792.0, + "7": 3126817792.0, + "8": 3126817792.0, + "9": 3126817792.0, + "10": 3126817792.0, + "11": 3126817792.0, + "12": 3126817792.0, + "13": 3146392576.0, + "14": 3152364544.0, + "15": 3152364544.0, + "16": 3152364544.0, + "17": 3152364544.0, + "18": 3152364544.0, + "19": 3152364544.0, + "20": 3152364544.0, + "21": 3152364544.0, + "22": 3152364544.0, + "23": 3152364544.0, + "24": 3152364544.0, + "25": 3152364544.0, + "26": 3152364544.0, + "27": 3152364544.0, + "28": 3152364544.0, + "29": 3152364544.0, + "30": 3152364544.0, + "31": 3152364544.0, + "32": 3152364544.0, + "33": 3152364544.0, + "34": 3152364544.0, + "35": 3152364544.0, + "36": 3152364544.0, + "37": 3152364544.0, + "38": 3152364544.0, + "39": 3152364544.0, + "40": 3152364544.0, + "41": 3152364544.0, + "42": 3152364544.0, + "43": 3152364544.0, + "44": 3152364544.0, + "45": 3152364544.0, + "46": 3152364544.0, + "47": 3152364544.0, + "48": 3152364544.0, + "49": 3152364544.0, + "50": 3152364544.0, + "51": 3152364544.0, + "52": 3152364544.0, + "53": 3152364544.0, + "54": 3152364544.0, + "55": 3152364544.0, + "56": 3152364544.0, + "57": 3152364544.0, + "58": 3152364544.0, + "59": 3152364544.0, + "60": 3152364544.0, + "61": 3152364544.0, + "62": 3152364544.0, + "63": 3152364544.0, + "64": 3152364544.0, + "65": 3152364544.0, + "66": 3152364544.0, + "67": 3152364544.0, + "68": 3152364544.0, + "69": 3152364544.0, + "70": 3152364544.0, + "71": 3152364544.0, + "72": 3152364544.0, + "73": 3152364544.0, + "74": 3152364544.0, + "75": 3152364544.0, + "76": 3152364544.0, + "77": 3152364544.0, + "78": 3152364544.0, + "79": 3152364544.0, + "80": 3152364544.0, + "81": 3152364544.0, + "82": 3152364544.0, + "83": 3152364544.0, + "84": 3152364544.0, + "85": 3152364544.0, + "86": 3152364544.0, + "87": 3152364544.0, + "88": 3152364544.0, + "89": 3152364544.0, + "90": 3152364544.0, + "91": 3152364544.0, + "92": 3152364544.0, + "93": 3152364544.0, + "94": 3152364544.0, + "95": 3152364544.0, + "96": 3152364544.0, + "97": 3152364544.0, + "98": 3152364544.0, + "99": 3152364544.0, + "100": 3152364544.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.3034, - "3": 0.24489, - "4": 0.20239, - "5": 0.18606, - "6": 0.188, - "7": 0.18113, - "8": 0.17161, - "9": 0.1761, - "10": 0.16774, - "11": 0.16475, - "12": 0.16531, - "13": 0.17147, - "14": 0.17118, - "15": 0.17561, - "16": 0.15896, - "17": 0.16639, - "18": 0.16698, - "19": 0.16202, - "20": 0.16347, - "21": 0.16853, - "22": 0.17029, - "23": 0.16304, - "24": 0.16791, - "25": 0.16793, - "26": 0.16289, - "27": 0.16937, - "28": 0.17246, - "29": 0.17111, - "30": 0.16906, - "31": 0.16921, - "32": 0.16458, - "33": 0.16956, - "34": 0.16061, - "35": 0.17198, - "36": 0.16363, - "37": 0.17617, - "38": 0.16352, - "39": 0.16111, - "40": 0.17286, - "41": 0.16569, - "42": 0.16388, - "43": 0.16457, - "44": 0.15938, - "45": 0.16019, - "46": 0.16147, - "47": 0.16054, - "48": 0.16252, - "49": 0.16599, - "50": 0.17642, - "51": 0.36082, - "52": 0.23725, - "53": 0.17696, - "54": 0.17435, - "55": 0.16475, - "56": 0.16318, - "57": 0.16321, - "58": 0.16181, - "59": 0.16521, - "60": 0.16167, - "61": 0.16151, - "62": 0.16406, - "63": 0.16501, - "64": 0.16193, - "65": 0.16318, - "66": 0.1689, - "67": 0.16405, - "68": 0.1602, - "69": 0.16818, - "70": 0.16761, - "71": 0.16686, - "72": 0.15876, - "73": 0.15964, - "74": 0.16608, - "75": 0.15741, - "76": 0.1673, - "77": 0.16306, - "78": 0.15921, - "79": 0.16069, - "80": 0.16039, - "81": 0.15802, - "82": 0.15834, - "83": 0.1601, - "84": 0.16156, - "85": 0.16097, - "86": 0.16922, - "87": 0.16335, - "88": 0.16125, - "89": 0.15443, - "90": 0.16107, - "91": 0.16218, - "92": 0.15801, - "93": 0.15843, - "94": 0.16221, - "95": 0.15979, - "96": 0.1583, - "97": 0.15628, - "98": 0.16181, - "99": 0.15683, - "100": 0.16117 + "2": 10.07331, + "3": 0.21396, + "4": 0.18077, + "5": 0.17988, + "6": 0.15588, + "7": 0.15793, + "8": 0.15174, + "9": 0.15636, + "10": 0.15027, + "11": 0.14783, + "12": 0.14169, + "13": 0.14957, + "14": 0.14575, + "15": 0.14808, + "16": 0.13539, + "17": 0.13851, + "18": 0.14162, + "19": 0.1405, + "20": 0.13936, + "21": 0.14159, + "22": 0.14241, + "23": 0.13728, + "24": 0.14137, + "25": 0.14679, + "26": 0.14482, + "27": 0.14612, + "28": 0.14299, + "29": 0.1428, + "30": 0.14665, + "31": 0.14455, + "32": 0.14111, + "33": 0.14201, + "34": 0.14055, + "35": 0.14068, + "36": 0.14545, + "37": 0.14568, + "38": 0.13545, + "39": 0.14332, + "40": 0.1426, + "41": 0.14439, + "42": 0.13614, + "43": 0.14138, + "44": 0.13768, + "45": 0.13543, + "46": 0.13929, + "47": 0.14394, + "48": 0.14067, + "49": 0.13868, + "50": 0.13729, + "51": 0.36115, + "52": 0.19983, + "53": 0.1468, + "54": 0.15298, + "55": 0.1395, + "56": 0.14196, + "57": 0.14207, + "58": 0.13925, + "59": 0.139, + "60": 0.14105, + "61": 0.1399, + "62": 0.13815, + "63": 0.14088, + "64": 0.14807, + "65": 0.13782, + "66": 0.1476, + "67": 0.14205, + "68": 0.1395, + "69": 0.14515, + "70": 0.1425, + "71": 0.1471, + "72": 0.13713, + "73": 0.1453, + "74": 0.13733, + "75": 0.13531, + "76": 0.14044, + "77": 0.13485, + "78": 0.1375, + "79": 0.1377, + "80": 0.13496, + "81": 0.13432, + "82": 0.13508, + "83": 0.13728, + "84": 0.13257, + "85": 0.1396, + "86": 0.13822, + "87": 0.14513, + "88": 0.13592, + "89": 0.1349, + "90": 0.13366, + "91": 0.13794, + "92": 0.14055, + "93": 0.13598, + "94": 0.13533, + "95": 0.13286, + "96": 0.13383, + "97": 0.13169, + "98": 0.13876, + "99": 0.13186, + "100": 0.13203 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json index 362ed2517a2..2da9238d706 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90649, - "4": 10.90498, - "5": 10.91356, - "6": 10.89668, - "7": 10.89772, - "8": 10.91234, - "9": 10.89371, - "10": 10.90081, - "11": 10.89194, - "12": 10.90692, - "13": 10.8718, - "14": 10.86067, - "15": 10.87356, - "16": 10.85943, - "17": 10.84008, - "18": 10.83747, - "19": 10.84048, - "20": 10.79175, - "21": 10.75636, - "22": 10.75648, - "23": 10.75817, - "24": 10.72019, - "25": 10.72096, - "26": 10.69808, - "27": 10.68267, - "28": 10.61119, - "29": 10.59082, - "30": 10.57447, - "31": 10.56111, - "32": 10.54229, - "33": 10.51733, - "34": 10.47715, - "35": 10.48521, - "36": 10.47888, - "37": 10.42825, - "38": 10.43511, - "39": 10.40608, - "40": 10.39417, - "41": 10.37114, - "42": 10.35538, - "43": 10.32417, - "44": 10.30788, - "45": 10.31573, - "46": 10.27767, - "47": 10.26813, - "48": 10.22377, - "49": 10.21599, - "50": 10.21902, - "51": 10.22193, - "52": 10.17292, - "53": 10.17695, + "1": 10.90111, + "2": 10.8957, + "3": 10.90626, + "4": 10.90562, + "5": 10.91338, + "6": 10.89684, + "7": 10.89792, + "8": 10.91285, + "9": 10.89279, + "10": 10.90027, + "11": 10.89132, + "12": 10.90672, + "13": 10.87062, + "14": 10.86054, + "15": 10.87324, + "16": 10.85911, + "17": 10.84015, + "18": 10.83726, + "19": 10.83943, + "20": 10.79178, + "21": 10.75607, + "22": 10.75557, + "23": 10.75782, + "24": 10.72, + "25": 10.72107, + "26": 10.69764, + "27": 10.68234, + "28": 10.61163, + "29": 10.5913, + "30": 10.57438, + "31": 10.56065, + "32": 10.54263, + "33": 10.51706, + "34": 10.47723, + "35": 10.48572, + "36": 10.47878, + "37": 10.42879, + "38": 10.43527, + "39": 10.40593, + "40": 10.39405, + "41": 10.3711, + "42": 10.35553, + "43": 10.32405, + "44": 10.3079, + "45": 10.3161, + "46": 10.27786, + "47": 10.26864, + "48": 10.22395, + "49": 10.21645, + "50": 10.21926, + "51": 10.22194, + "52": 10.17303, + "53": 10.17725, "54": 10.14489, - "55": 10.11386, - "56": 10.1422, - "57": 10.12634, - "58": 10.13387, - "59": 10.07883, - "60": 10.10032, - "61": 10.0569, - "62": 10.02093, - "63": 10.0933, - "64": 10.03529, - "65": 10.00596, - "66": 10.04629, - "67": 10.01887, - "68": 9.97911, - "69": 9.99395, - "70": 9.98094, - "71": 10.00401, - "72": 9.98666, - "73": 9.96193, - "74": 9.9599, - "75": 9.91413, - "76": 9.95283, - "77": 9.95124, - "78": 9.90005, - "79": 9.90154, - "80": 9.91121, - "81": 9.94218, - "82": 9.89783, - "83": 9.85125, - "84": 9.79584, - "85": 9.77428, - "86": 9.89304, - "87": 9.90528, - "88": 9.883, - "89": 9.82604, - "90": 9.8253, - "91": 9.82873, - "92": 9.82373, - "93": 9.75213, - "94": 9.83157, - "95": 9.82108, - "96": 9.80782, - "97": 9.75363, - "98": 9.78415, - "99": 9.83389, - "100": 9.72314 + "55": 10.1139, + "56": 10.14246, + "57": 10.12615, + "58": 10.13392, + "59": 10.07842, + "60": 10.10018, + "61": 10.05693, + "62": 10.02102, + "63": 10.09347, + "64": 10.03541, + "65": 10.00572, + "66": 10.04666, + "67": 10.01884, + "68": 9.9791, + "69": 9.99387, + "70": 9.9811, + "71": 10.00411, + "72": 9.9866, + "73": 9.96195, + "74": 9.96004, + "75": 9.91415, + "76": 9.95265, + "77": 9.9512, + "78": 9.89997, + "79": 9.90165, + "80": 9.91122, + "81": 9.94213, + "82": 9.89797, + "83": 9.85138, + "84": 9.79603, + "85": 9.77409, + "86": 9.89333, + "87": 9.90529, + "88": 9.88326, + "89": 9.82622, + "90": 9.82545, + "91": 9.82891, + "92": 9.8238, + "93": 9.7521, + "94": 9.83174, + "95": 9.82115, + "96": 9.80774, + "97": 9.75361, + "98": 9.78431, + "99": 9.83399, + "100": 9.72316 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1363.0, - "4": 1284.0, - "5": 1232.0, - "6": 1243.0, - "7": 1313.0, - "8": 1225.0, - "9": 1225.0, - "10": 1193.0, - "11": 1184.0, - "12": 1158.0, - "13": 1384.0, - "14": 1251.0, - "15": 1118.0, - "16": 1300.0, - "17": 1330.0, - "18": 1189.0, - "19": 1107.0, - "20": 1077.0, - "21": 1136.0, - "22": 1123.0, - "23": 1292.0, - "24": 1187.0, - "25": 1126.0, - "26": 1256.0, - "27": 1218.0, - "28": 1134.0, - "29": 1166.0, - "30": 1154.0, - "31": 1151.0, - "32": 1176.0, - "33": 1276.0, - "34": 1108.0, - "35": 1151.0, - "36": 1134.0, - "37": 1210.0, - "38": 1192.0, - "39": 1286.0, - "40": 1221.0, - "41": 1170.0, - "42": 1177.0, - "43": 1149.0, - "44": 1224.0, - "45": 1204.0, - "46": 1271.0, - "47": 1239.0, - "48": 1307.0, - "49": 1225.0, - "50": 1288.0, - "51": 1461.0, - "52": 1330.0, - "53": 1286.0, - "54": 1422.0, - "55": 1193.0, - "56": 1364.0, - "57": 1231.0, - "58": 1050746.0, - "59": 1386.0, - "60": 1398.0, - "61": 42279.0, - "62": 49536.0, - "63": 1562.0, - "64": 42506.0, - "65": 1237.0, - "66": 1050485.0, - "67": 50645.0, - "68": 1553.0, - "69": 5459.0, - "70": 6617.0, - "71": 2356.0, - "72": 57739.0, - "73": 57912.0, - "74": 67070.0, - "75": 11773.0, - "76": 70154.0, - "77": 1701.0, - "78": 1612.0, - "79": 6649.0, - "80": 5470.0, - "81": 2645.0, - "82": 2709.0, - "83": 1410.0, - "84": 1050554.0, - "85": 1560.0, - "86": 1540.0, - "87": 2152913.0, - "88": 2420.0, - "89": 10807.0, - "90": 1060087.0, - "91": 1295.0, - "92": 2099726.0, - "93": 10800.0, - "94": 2764.0, - "95": 1050655.0, - "96": 116340.0, - "97": 1050841.0, - "98": 54863.0, - "99": 51572.0, - "100": 1050668.0 + "1": 1133.0, + "2": 1120.0, + "3": 1357.0, + "4": 1328.0, + "5": 1243.0, + "6": 1194.0, + "7": 1359.0, + "8": 1220.0, + "9": 1207.0, + "10": 1187.0, + "11": 1241.0, + "12": 1150.0, + "13": 1293.0, + "14": 1250.0, + "15": 1165.0, + "16": 1281.0, + "17": 1240.0, + "18": 1254.0, + "19": 1207.0, + "20": 1123.0, + "21": 1158.0, + "22": 1122.0, + "23": 1212.0, + "24": 1223.0, + "25": 1195.0, + "26": 1229.0, + "27": 1201.0, + "28": 1173.0, + "29": 1129.0, + "30": 1179.0, + "31": 1078.0, + "32": 1148.0, + "33": 1241.0, + "34": 1120.0, + "35": 1181.0, + "36": 1113.0, + "37": 1150.0, + "38": 1171.0, + "39": 1185.0, + "40": 1211.0, + "41": 1313.0, + "42": 1187.0, + "43": 1181.0, + "44": 1146.0, + "45": 1186.0, + "46": 1210.0, + "47": 1153.0, + "48": 1245.0, + "49": 1302.0, + "50": 1326.0, + "51": 1447.0, + "52": 1342.0, + "53": 1269.0, + "54": 1381.0, + "55": 1228.0, + "56": 1273.0, + "57": 1214.0, + "58": 50777.0, + "59": 1361.0, + "60": 1406.0, + "61": 42322.0, + "62": 49580.0, + "63": 1577.0, + "64": 43545.0, + "65": 1216.0, + "66": 1050479.0, + "67": 51707.0, + "68": 1528.0, + "69": 5467.0, + "70": 6590.0, + "71": 2418.0, + "72": 57795.0, + "73": 7777.0, + "74": 11706.0, + "75": 13832.0, + "76": 70208.0, + "77": 1681.0, + "78": 1563.0, + "79": 6703.0, + "80": 5581.0, + "81": 2703.0, + "82": 2750.0, + "83": 1486.0, + "84": 1050539.0, + "85": 1435.0, + "86": 1495.0, + "87": 2151913.0, + "88": 2377.0, + "89": 10766.0, + "90": 1060091.0, + "91": 1258.0, + "92": 2099760.0, + "93": 11822.0, + "94": 1776.0, + "95": 1050670.0, + "96": 116325.0, + "97": 53923.0, + "98": 53892.0, + "99": 5442.0, + "100": 1050643.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 788519424.0, - "2": 788509696.0, - "3": 788487680.0, - "4": 788466176.0, - "5": 788516352.0, - "6": 788500992.0, - "7": 788425216.0, - "8": 788463104.0, - "9": 788467200.0, - "10": 788497920.0, - "11": 788475904.0, - "12": 788486144.0, - "13": 788488704.0, - "14": 788493824.0, - "15": 788476928.0, - "16": 788465152.0, + "1": 788519936.0, + "2": 788510720.0, + "3": 788485120.0, + "4": 788465664.0, + "5": 788514816.0, + "6": 788501504.0, + "7": 788426240.0, + "8": 788464128.0, + "9": 788466176.0, + "10": 788499456.0, + "11": 788475392.0, + "12": 788483584.0, + "13": 788490240.0, + "14": 788493312.0, + "15": 788476416.0, + "16": 788467712.0, "17": 788476416.0, - "18": 788482560.0, - "19": 788512768.0, - "20": 788488192.0, - "21": 788554752.0, - "22": 788592128.0, - "23": 788578816.0, + "18": 788483584.0, + "19": 788512256.0, + "20": 788488704.0, + "21": 788551680.0, + "22": 788594176.0, + "23": 788579328.0, "24": 788489728.0, - "25": 788579328.0, - "26": 788623360.0, - "27": 788565504.0, - "28": 788536320.0, - "29": 788600832.0, + "25": 788578816.0, + "26": 788620288.0, + "27": 788563456.0, + "28": 788538368.0, + "29": 788597760.0, "30": 788569600.0, - "31": 788587520.0, - "32": 788580352.0, + "31": 788587008.0, + "32": 788580864.0, "33": 788574208.0, - "34": 788564992.0, - "35": 788553728.0, - "36": 788604416.0, - "37": 788551680.0, - "38": 788576768.0, - "39": 788595712.0, - "40": 788554240.0, - "41": 788546560.0, - "42": 788506112.0, - "43": 788538368.0, - "44": 788487168.0, - "45": 788457984.0, - "46": 788498944.0, - "47": 788450816.0, - "48": 788437504.0, - "49": 788416000.0, - "50": 788386816.0, - "51": 788401664.0, - "52": 788369408.0, - "53": 788322816.0, - "54": 788357120.0, - "55": 788350464.0, - "56": 788314624.0, - "57": 788388864.0, - "58": 788365312.0, + "34": 788563456.0, + "35": 788553216.0, + "36": 788605440.0, + "37": 788550144.0, + "38": 788575744.0, + "39": 788595200.0, + "40": 788552704.0, + "41": 788545536.0, + "42": 788503552.0, + "43": 788534272.0, + "44": 788486144.0, + "45": 788453888.0, + "46": 788497408.0, + "47": 788449792.0, + "48": 788434944.0, + "49": 788413952.0, + "50": 788384256.0, + "51": 788398592.0, + "52": 788370944.0, + "53": 788320256.0, + "54": 788355584.0, + "55": 788348416.0, + "56": 788313088.0, + "57": 788386816.0, + "58": 788362752.0, "59": 788348416.0, - "60": 788370944.0, - "61": 788306944.0, - "62": 788299264.0, - "63": 788331008.0, - "64": 788336640.0, - "65": 788324864.0, - "66": 788332032.0, - "67": 788337664.0, - "68": 788326912.0, - "69": 788331008.0, - "70": 788350976.0, + "60": 788368896.0, + "61": 788305920.0, + "62": 788297728.0, + "63": 788329472.0, + "64": 788335104.0, + "65": 788324352.0, + "66": 788331008.0, + "67": 788335104.0, + "68": 788327424.0, + "69": 788331520.0, + "70": 788349952.0, "71": 788344832.0, - "72": 788391424.0, - "73": 788350976.0, - "74": 788380672.0, - "75": 788320256.0, - "76": 788292096.0, - "77": 788396544.0, - "78": 788358656.0, - "79": 788318208.0, - "80": 788278272.0, - "81": 788282368.0, - "82": 788242432.0, - "83": 788274688.0, + "72": 788392960.0, + "73": 788349952.0, + "74": 788381184.0, + "75": 788319744.0, + "76": 788291584.0, + "77": 788393472.0, + "78": 788356096.0, + "79": 788316160.0, + "80": 788275712.0, + "81": 788283904.0, + "82": 788240896.0, + "83": 788277248.0, "84": 788317696.0, - "85": 788315136.0, - "86": 788338176.0, - "87": 788144640.0, - "88": 788254208.0, - "89": 788261888.0, - "90": 788246016.0, - "91": 788318208.0, - "92": 788240384.0, - "93": 788244480.0, - "94": 788385792.0, - "95": 788250112.0, - "96": 788293632.0, - "97": 788358656.0, - "98": 788327936.0, - "99": 788265472.0, - "100": 788357632.0 + "85": 788314624.0, + "86": 788337664.0, + "87": 788147200.0, + "88": 788256256.0, + "89": 788264448.0, + "90": 788247552.0, + "91": 788319232.0, + "92": 788239872.0, + "93": 788245504.0, + "94": 788387840.0, + "95": 788252672.0, + "96": 788294656.0, + "97": 788360192.0, + "98": 788326912.0, + "99": 788267008.0, + "100": 788357120.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3122096128.0, - "2": 3285577216.0, - "3": 3285577216.0, - "4": 3285577216.0, - "5": 3288475648.0, - "6": 3288475648.0, - "7": 3288475648.0, - "8": 3288475648.0, - "9": 3288475648.0, - "10": 3288475648.0, - "11": 3288475648.0, - "12": 3288475648.0, - "13": 3288475648.0, - "14": 3288475648.0, - "15": 3288475648.0, - "16": 3288475648.0, - "17": 3288475648.0, - "18": 3288475648.0, - "19": 3288475648.0, - "20": 3288475648.0, - "21": 3315479040.0, - "22": 3342354432.0, - "23": 3342354432.0, - "24": 3342354432.0, - "25": 3342354432.0, - "26": 3359620608.0, - "27": 3359620608.0, - "28": 3359620608.0, - "29": 3359620608.0, - "30": 3359620608.0, - "31": 3359620608.0, - "32": 3359620608.0, - "33": 3359620608.0, - "34": 3359620608.0, - "35": 3359620608.0, - "36": 3359620608.0, - "37": 3359620608.0, - "38": 3359620608.0, - "39": 3359620608.0, - "40": 3359620608.0, - "41": 3359620608.0, - "42": 3359620608.0, - "43": 3359620608.0, - "44": 3359620608.0, - "45": 3359620608.0, - "46": 3359620608.0, - "47": 3359620608.0, - "48": 3359620608.0, - "49": 3359620608.0, - "50": 3359620608.0, - "51": 3359620608.0, - "52": 3359620608.0, - "53": 3359620608.0, - "54": 3359620608.0, - "55": 3359620608.0, - "56": 3359620608.0, - "57": 3359620608.0, - "58": 3359620608.0, - "59": 3359620608.0, - "60": 3359620608.0, - "61": 3359620608.0, - "62": 3359620608.0, - "63": 3359620608.0, - "64": 3359620608.0, - "65": 3359620608.0, - "66": 3359620608.0, - "67": 3359620608.0, - "68": 3359620608.0, - "69": 3359620608.0, - "70": 3359620608.0, - "71": 3359620608.0, - "72": 3359620608.0, - "73": 3359620608.0, - "74": 3359620608.0, - "75": 3359620608.0, - "76": 3359620608.0, - "77": 3359620608.0, - "78": 3359620608.0, - "79": 3359620608.0, - "80": 3359620608.0, - "81": 3359620608.0, - "82": 3359620608.0, - "83": 3359620608.0, - "84": 3359620608.0, - "85": 3359620608.0, - "86": 3359620608.0, - "87": 3359620608.0, - "88": 3359620608.0, - "89": 3359620608.0, - "90": 3359620608.0, - "91": 3359620608.0, - "92": 3359620608.0, - "93": 3359620608.0, - "94": 3359620608.0, - "95": 3359620608.0, - "96": 3359620608.0, - "97": 3359620608.0, - "98": 3359620608.0, - "99": 3359620608.0, - "100": 3359620608.0 + "1": 3121236480.0, + "2": 3284394496.0, + "3": 3284394496.0, + "4": 3284394496.0, + "5": 3289592320.0, + "6": 3289592320.0, + "7": 3289592320.0, + "8": 3289592320.0, + "9": 3289592320.0, + "10": 3289592320.0, + "11": 3289592320.0, + "12": 3289592320.0, + "13": 3289592320.0, + "14": 3289592320.0, + "15": 3289592320.0, + "16": 3289592320.0, + "17": 3289592320.0, + "18": 3289592320.0, + "19": 3289592320.0, + "20": 3289592320.0, + "21": 3315035648.0, + "22": 3342148608.0, + "23": 3342148608.0, + "24": 3342148608.0, + "25": 3342148608.0, + "26": 3359425024.0, + "27": 3359425024.0, + "28": 3359425024.0, + "29": 3359425024.0, + "30": 3359425024.0, + "31": 3359425024.0, + "32": 3359425024.0, + "33": 3359425024.0, + "34": 3359425024.0, + "35": 3359425024.0, + "36": 3359425024.0, + "37": 3359425024.0, + "38": 3359425024.0, + "39": 3359425024.0, + "40": 3359425024.0, + "41": 3359425024.0, + "42": 3359425024.0, + "43": 3359425024.0, + "44": 3359425024.0, + "45": 3359425024.0, + "46": 3359425024.0, + "47": 3359425024.0, + "48": 3359425024.0, + "49": 3359425024.0, + "50": 3359425024.0, + "51": 3359425024.0, + "52": 3359425024.0, + "53": 3359425024.0, + "54": 3359425024.0, + "55": 3359425024.0, + "56": 3359425024.0, + "57": 3359425024.0, + "58": 3359425024.0, + "59": 3359425024.0, + "60": 3359425024.0, + "61": 3359425024.0, + "62": 3359425024.0, + "63": 3359425024.0, + "64": 3359425024.0, + "65": 3359425024.0, + "66": 3359425024.0, + "67": 3359425024.0, + "68": 3359425024.0, + "69": 3359425024.0, + "70": 3359425024.0, + "71": 3359425024.0, + "72": 3359425024.0, + "73": 3359425024.0, + "74": 3359425024.0, + "75": 3359425024.0, + "76": 3359425024.0, + "77": 3359425024.0, + "78": 3359425024.0, + "79": 3359425024.0, + "80": 3359425024.0, + "81": 3359425024.0, + "82": 3359425024.0, + "83": 3359425024.0, + "84": 3359425024.0, + "85": 3359425024.0, + "86": 3359425024.0, + "87": 3359425024.0, + "88": 3359425024.0, + "89": 3359425024.0, + "90": 3359425024.0, + "91": 3359425024.0, + "92": 3359425024.0, + "93": 3359425024.0, + "94": 3359425024.0, + "95": 3359425024.0, + "96": 3359425024.0, + "97": 3359425024.0, + "98": 3359425024.0, + "99": 3359425024.0, + "100": 3359425024.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.18868, - "3": 0.18976, - "4": 0.18601, - "5": 0.178, - "6": 0.17013, - "7": 0.17369, - "8": 0.18316, - "9": 0.16018, - "10": 0.1641, - "11": 0.17079, - "12": 0.16346, - "13": 0.1628, - "14": 0.16331, - "15": 0.15023, - "16": 0.15341, - "17": 0.15639, - "18": 0.15465, - "19": 0.14892, - "20": 0.15916, - "21": 0.15901, - "22": 0.15961, - "23": 0.15706, - "24": 0.16439, - "25": 0.17181, - "26": 0.15735, - "27": 0.17408, - "28": 0.17844, - "29": 0.17656, - "30": 0.1684, - "31": 0.15774, - "32": 0.18388, - "33": 0.17033, - "34": 0.17365, - "35": 0.16585, - "36": 0.17175, - "37": 0.16836, - "38": 0.16953, - "39": 0.17684, - "40": 0.1544, - "41": 0.16069, - "42": 0.16297, - "43": 0.17956, - "44": 0.16333, - "45": 0.16491, - "46": 0.15417, - "47": 0.15747, - "48": 0.15582, - "49": 0.15871, - "50": 0.15837, - "51": 0.20584, - "52": 0.19572, - "53": 0.15813, - "54": 0.15301, - "55": 0.15257, - "56": 0.14771, - "57": 0.16512, - "58": 0.15268, - "59": 0.15716, - "60": 0.16019, - "61": 0.15621, - "62": 0.16853, - "63": 0.15277, - "64": 0.14379, - "65": 0.14892, - "66": 0.15734, - "67": 0.15021, - "68": 0.15207, - "69": 0.15622, - "70": 0.14784, - "71": 0.14597, - "72": 0.15762, - "73": 0.15952, - "74": 0.15146, - "75": 0.15538, - "76": 0.14898, - "77": 0.15001, - "78": 0.1499, - "79": 0.15411, - "80": 0.1511, - "81": 0.14847, - "82": 0.14689, - "83": 0.14637, - "84": 0.14672, - "85": 0.14407, - "86": 0.15642, - "87": 0.14024, - "88": 0.15645, - "89": 0.14898, - "90": 0.14543, - "91": 0.14593, - "92": 0.14214, - "93": 0.14251, - "94": 0.14648, - "95": 0.15641, - "96": 0.14005, - "97": 0.14035, - "98": 0.14839, - "99": 0.12987, - "100": 0.13898 + "2": 4.9135, + "3": 0.19417, + "4": 0.17207, + "5": 0.18219, + "6": 0.16543, + "7": 0.1774, + "8": 0.18611, + "9": 0.17809, + "10": 0.16581, + "11": 0.17435, + "12": 0.15833, + "13": 0.16875, + "14": 0.16858, + "15": 0.15588, + "16": 0.16486, + "17": 0.17278, + "18": 0.15113, + "19": 0.15841, + "20": 0.16148, + "21": 0.17064, + "22": 0.16443, + "23": 0.15727, + "24": 0.1617, + "25": 0.18126, + "26": 0.1563, + "27": 0.17164, + "28": 0.18155, + "29": 0.17086, + "30": 0.17049, + "31": 0.17088, + "32": 0.1818, + "33": 0.17985, + "34": 0.18336, + "35": 0.1592, + "36": 0.17494, + "37": 0.17986, + "38": 0.17697, + "39": 0.18571, + "40": 0.16652, + "41": 0.16481, + "42": 0.17001, + "43": 0.1717, + "44": 0.16769, + "45": 0.15874, + "46": 0.17204, + "47": 0.16377, + "48": 0.14596, + "49": 0.15439, + "50": 0.16224, + "51": 0.21157, + "52": 0.20217, + "53": 0.16749, + "54": 0.15649, + "55": 0.15512, + "56": 0.14461, + "57": 0.16486, + "58": 0.1654, + "59": 0.15685, + "60": 0.16367, + "61": 0.14272, + "62": 0.17682, + "63": 0.15365, + "64": 0.14932, + "65": 0.15659, + "66": 0.14867, + "67": 0.15655, + "68": 0.14334, + "69": 0.15562, + "70": 0.15061, + "71": 0.15262, + "72": 0.15289, + "73": 0.15784, + "74": 0.15757, + "75": 0.15612, + "76": 0.15125, + "77": 0.1571, + "78": 0.1548, + "79": 0.15, + "80": 0.14862, + "81": 0.14915, + "82": 0.14682, + "83": 0.15651, + "84": 0.15505, + "85": 0.15062, + "86": 0.15217, + "87": 0.14306, + "88": 0.15313, + "89": 0.14982, + "90": 0.14149, + "91": 0.15121, + "92": 0.13782, + "93": 0.14528, + "94": 0.14496, + "95": 0.14474, + "96": 0.14134, + "97": 0.15083, + "98": 0.14352, + "99": 0.14785, + "100": 0.14678 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index c31c834f04c..a4472e113a3 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.92513, - "4": 10.91739, - "5": 10.9269, - "6": 10.92031, - "7": 10.91598, - "8": 10.91649, - "9": 10.93519, - "10": 10.9206, - "11": 10.90717, - "12": 10.91306, - "13": 10.92203, - "14": 10.90168, - "15": 10.89219, - "16": 10.87582, - "17": 10.88267, - "18": 10.85858, - "19": 10.87882, - "20": 10.80364, - "21": 10.81054, - "22": 10.78654, - "23": 10.77822, - "24": 10.74148, - "25": 10.73996, - "26": 10.73202, - "27": 10.71123, - "28": 10.65525, - "29": 10.62359, - "30": 10.59735, - "31": 10.58089, - "32": 10.57395, - "33": 10.54555, - "34": 10.50091, - "35": 10.51208, - "36": 10.48236, - "37": 10.45341, - "38": 10.46938, - "39": 10.42943, - "40": 10.41557, - "41": 10.3926, - "42": 10.38242, - "43": 10.34965, - "44": 10.33277, - "45": 10.33406, - "46": 10.30638, - "47": 10.28401, - "48": 10.2469, - "49": 10.2361, - "50": 10.24807, - "51": 10.24514, - "52": 10.19574, - "53": 10.19181, - "54": 10.16588, - "55": 10.12867, - "56": 10.16541, - "57": 10.13627, - "58": 10.15663, - "59": 10.10169, - "60": 10.1085, - "61": 10.07239, - "62": 10.03691, - "63": 10.10933, - "64": 10.05782, - "65": 10.02658, - "66": 10.05862, - "67": 10.02843, - "68": 9.98898, - "69": 10.00162, - "70": 9.99443, - "71": 10.02546, - "72": 9.98142, - "73": 9.9673, - "74": 9.95642, - "75": 9.92751, - "76": 9.97435, - "77": 9.96221, - "78": 9.9049, - "79": 9.91517, - "80": 9.92522, - "81": 9.94341, - "82": 9.90375, - "83": 9.85642, - "84": 9.79325, - "85": 9.77579, - "86": 9.89058, - "87": 9.9164, - "88": 9.89322, - "89": 9.82868, - "90": 9.82075, - "91": 9.83171, - "92": 9.82545, - "93": 9.75863, - "94": 9.83386, - "95": 9.83104, - "96": 9.81382, - "97": 9.75302, - "98": 9.78874, - "99": 9.8339, - "100": 9.72866 + "1": 10.93064, + "2": 10.92264, + "3": 10.92441, + "4": 10.91686, + "5": 10.92701, + "6": 10.92079, + "7": 10.91574, + "8": 10.91647, + "9": 10.93458, + "10": 10.92054, + "11": 10.90683, + "12": 10.91327, + "13": 10.9214, + "14": 10.90157, + "15": 10.89311, + "16": 10.87614, + "17": 10.88256, + "18": 10.85905, + "19": 10.87765, + "20": 10.80275, + "21": 10.81075, + "22": 10.78656, + "23": 10.77818, + "24": 10.74186, + "25": 10.73977, + "26": 10.73189, + "27": 10.71152, + "28": 10.65527, + "29": 10.62335, + "30": 10.59679, + "31": 10.58051, + "32": 10.57392, + "33": 10.5458, + "34": 10.5016, + "35": 10.51196, + "36": 10.48208, + "37": 10.45398, + "38": 10.46925, + "39": 10.42854, + "40": 10.41547, + "41": 10.39223, + "42": 10.38216, + "43": 10.34944, + "44": 10.33256, + "45": 10.33374, + "46": 10.30632, + "47": 10.2841, + "48": 10.24662, + "49": 10.23591, + "50": 10.24832, + "51": 10.24533, + "52": 10.19596, + "53": 10.19195, + "54": 10.16609, + "55": 10.12861, + "56": 10.16556, + "57": 10.13618, + "58": 10.15698, + "59": 10.10159, + "60": 10.10852, + "61": 10.07214, + "62": 10.03671, + "63": 10.10923, + "64": 10.05778, + "65": 10.02626, + "66": 10.05854, + "67": 10.02842, + "68": 9.98887, + "69": 10.00132, + "70": 9.99481, + "71": 10.02538, + "72": 9.98121, + "73": 9.96714, + "74": 9.95643, + "75": 9.92737, + "76": 9.97455, + "77": 9.96204, + "78": 9.90473, + "79": 9.91522, + "80": 9.92511, + "81": 9.94349, + "82": 9.9036, + "83": 9.85656, + "84": 9.79289, + "85": 9.77545, + "86": 9.89051, + "87": 9.91663, + "88": 9.89297, + "89": 9.82866, + "90": 9.82093, + "91": 9.83177, + "92": 9.82526, + "93": 9.75885, + "94": 9.8339, + "95": 9.83071, + "96": 9.8138, + "97": 9.75301, + "98": 9.7888, + "99": 9.83387, + "100": 9.72864 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 48952.0, - "4": 48834.0, - "5": 50218.0, - "6": 50021.0, - "7": 51375.0, - "8": 49745.0, - "9": 51326.0, - "10": 52144.0, - "11": 49913.0, - "12": 48458.0, - "13": 51932.0, - "14": 50123.0, - "15": 47949.0, - "16": 49843.0, - "17": 49882.0, - "18": 52196.0, - "19": 51092.0, - "20": 52823.0, - "21": 49874.0, - "22": 49685.0, - "23": 53705.0, - "24": 52981.0, - "25": 49745.0, - "26": 53405.0, - "27": 53055.0, - "28": 49700.0, - "29": 52803.0, - "30": 54243.0, - "31": 56717.0, - "32": 53359.0, - "33": 56818.0, - "34": 53753.0, - "35": 55935.0, - "36": 57567.0, - "37": 54585.0, - "38": 55711.0, - "39": 58747.0, - "40": 58084.0, - "41": 55456.0, - "42": 57851.0, - "43": 61712.0, - "44": 58241.0, - "45": 67706.0, - "46": 57622.0, - "47": 105431.0, - "48": 64577.0, - "49": 71094.0, - "50": 67123.0, - "51": 63642.0, - "52": 71395.0, - "53": 110439.0, - "54": 72921.0, - "55": 61905.0, - "56": 1115059.0, - "57": 105088.0, - "58": 1127666.0, - "59": 80324.0, - "60": 123288.0, - "61": 1114978.0, - "62": 2170919.0, - "63": 70744.0, - "64": 91071.0, - "65": 173088.0, - "66": 1120234.0, - "67": 139759.0, - "68": 1123171.0, - "69": 1163608.0, - "70": 193228.0, - "71": 1118079.0, - "72": 2177823.0, - "73": 1182410.0, - "74": 2166520.0, - "75": 2166182.0, - "76": 1129053.0, - "77": 2168343.0, - "78": 1177493.0, - "79": 1175756.0, - "80": 1176743.0, - "81": 2170930.0, - "82": 2170395.0, - "83": 2170477.0, - "84": 1122782.0, - "85": 144101.0, - "86": 1124182.0, - "87": 1171244.0, - "88": 1168022.0, - "89": 1184028.0, - "90": 1183941.0, - "91": 1164459.0, - "92": 1131381.0, - "93": 1120131.0, - "94": 2175177.0, - "95": 2225361.0, - "96": 2172323.0, - "97": 2176261.0, - "98": 2175520.0, - "99": 1172004.0, - "100": 1197756.0 + "1": 49180.0, + "2": 50476.0, + "3": 49242.0, + "4": 49191.0, + "5": 50171.0, + "6": 50251.0, + "7": 51448.0, + "8": 49384.0, + "9": 50604.0, + "10": 52259.0, + "11": 50259.0, + "12": 48576.0, + "13": 51717.0, + "14": 50076.0, + "15": 48052.0, + "16": 49556.0, + "17": 49957.0, + "18": 51611.0, + "19": 50958.0, + "20": 52961.0, + "21": 50270.0, + "22": 50221.0, + "23": 54450.0, + "24": 52688.0, + "25": 49964.0, + "26": 53412.0, + "27": 52953.0, + "28": 50006.0, + "29": 52765.0, + "30": 54045.0, + "31": 56577.0, + "32": 53583.0, + "33": 56926.0, + "34": 53619.0, + "35": 55954.0, + "36": 57810.0, + "37": 54727.0, + "38": 55194.0, + "39": 59013.0, + "40": 58378.0, + "41": 55667.0, + "42": 58064.0, + "43": 61631.0, + "44": 57353.0, + "45": 68423.0, + "46": 58277.0, + "47": 105279.0, + "48": 68157.0, + "49": 70699.0, + "50": 67311.0, + "51": 63481.0, + "52": 71358.0, + "53": 110422.0, + "54": 72729.0, + "55": 110560.0, + "56": 1115272.0, + "57": 64834.0, + "58": 125706.0, + "59": 80693.0, + "60": 1116131.0, + "61": 1115149.0, + "62": 1126726.0, + "63": 71035.0, + "64": 146028.0, + "65": 173156.0, + "66": 1120354.0, + "67": 190880.0, + "68": 1122978.0, + "69": 1164011.0, + "70": 193177.0, + "71": 1118204.0, + "72": 2177337.0, + "73": 1181882.0, + "74": 2166710.0, + "75": 2165814.0, + "76": 1128584.0, + "77": 2168839.0, + "78": 1177721.0, + "79": 1176109.0, + "80": 1176668.0, + "81": 2170594.0, + "82": 1174064.0, + "83": 2170688.0, + "84": 1123127.0, + "85": 197488.0, + "86": 1124226.0, + "87": 1171138.0, + "88": 2169625.0, + "89": 1184133.0, + "90": 1183124.0, + "91": 1218390.0, + "92": 1131755.0, + "93": 1127776.0, + "94": 2175430.0, + "95": 2173930.0, + "96": 2169269.0, + "97": 2176014.0, + "98": 2175927.0, + "99": 1171444.0, + "100": 1197824.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1059404800.0, - "2": 1059426304.0, - "3": 1059409408.0, + "1": 1059406336.0, + "2": 1059425280.0, + "3": 1059410432.0, "4": 1059407360.0, - "5": 1059418624.0, - "6": 1059404288.0, - "7": 1059398656.0, + "5": 1059418112.0, + "6": 1059404800.0, + "7": 1059400704.0, "8": 1059425280.0, "9": 1059375104.0, "10": 1059374080.0, "11": 1059355648.0, - "12": 1059381248.0, - "13": 1059381248.0, - "14": 1059368448.0, + "12": 1059383808.0, + "13": 1059383296.0, + "14": 1059369984.0, "15": 1059378688.0, - "16": 1059364864.0, - "17": 1059371520.0, - "18": 1059343872.0, - "19": 1059335680.0, - "20": 1059316224.0, - "21": 1059302912.0, - "22": 1059234304.0, + "16": 1059364352.0, + "17": 1059372544.0, + "18": 1059341824.0, + "19": 1059337728.0, + "20": 1059316736.0, + "21": 1059302400.0, + "22": 1059233280.0, "23": 1059273216.0, - "24": 1059272704.0, - "25": 1059232768.0, - "26": 1059252736.0, - "27": 1059254272.0, - "28": 1059177984.0, - "29": 1059168256.0, - "30": 1059190272.0, - "31": 1059146752.0, - "32": 1059154944.0, - "33": 1059131392.0, - "34": 1059156992.0, - "35": 1059128320.0, - "36": 1059153920.0, - "37": 1059128320.0, - "38": 1059129344.0, - "39": 1059120128.0, + "24": 1059275264.0, + "25": 1059232256.0, + "26": 1059250688.0, + "27": 1059253760.0, + "28": 1059179520.0, + "29": 1059169792.0, + "30": 1059188736.0, + "31": 1059148288.0, + "32": 1059156480.0, + "33": 1059130368.0, + "34": 1059158016.0, + "35": 1059126272.0, + "36": 1059152896.0, + "37": 1059129344.0, + "38": 1059130368.0, + "39": 1059122176.0, "40": 1059153408.0, - "41": 1059121152.0, - "42": 1059118592.0, - "43": 1059116032.0, - "44": 1059122176.0, - "45": 1059144192.0, - "46": 1059119104.0, - "47": 1059133440.0, - "48": 1059131392.0, - "49": 1059138560.0, - "50": 1059123712.0, - "51": 1059121664.0, - "52": 1059122688.0, - "53": 1059100160.0, - "54": 1059146752.0, - "55": 1059116544.0, - "56": 1059119104.0, - "57": 1059092992.0, - "58": 1059114496.0, - "59": 1059089920.0, - "60": 1059072000.0, - "61": 1059088384.0, - "62": 1059093504.0, - "63": 1059085312.0, - "64": 1059065856.0, - "65": 1059073024.0, - "66": 1059051520.0, - "67": 1059049984.0, - "68": 1059025408.0, - "69": 1059041280.0, + "41": 1059123712.0, + "42": 1059117056.0, + "43": 1059117056.0, + "44": 1059122688.0, + "45": 1059145728.0, + "46": 1059118592.0, + "47": 1059131904.0, + "48": 1059131904.0, + "49": 1059140096.0, + "50": 1059124736.0, + "51": 1059123200.0, + "52": 1059125248.0, + "53": 1059101696.0, + "54": 1059148800.0, + "55": 1059118592.0, + "56": 1059121152.0, + "57": 1059094528.0, + "58": 1059115008.0, + "59": 1059092480.0, + "60": 1059071488.0, + "61": 1059089408.0, + "62": 1059096064.0, + "63": 1059086336.0, + "64": 1059067392.0, + "65": 1059074048.0, + "66": 1059052544.0, + "67": 1059052544.0, + "68": 1059028480.0, + "69": 1059041792.0, "70": 1059020288.0, "71": 1059035136.0, - "72": 1059006464.0, - "73": 1059009536.0, - "74": 1059009024.0, - "75": 1059028480.0, - "76": 1059019264.0, - "77": 1059038208.0, - "78": 1059022848.0, + "72": 1059005952.0, + "73": 1059011072.0, + "74": 1059010560.0, + "75": 1059030016.0, + "76": 1059020288.0, + "77": 1059036160.0, + "78": 1059024896.0, "79": 1059030016.0, "80": 1059014656.0, - "81": 1059033600.0, - "82": 1059005952.0, - "83": 1059012096.0, - "84": 1059024384.0, - "85": 1059003904.0, - "86": 1059016192.0, - "87": 1059000320.0, - "88": 1058989056.0, + "81": 1059033088.0, + "82": 1059005440.0, + "83": 1059007488.0, + "84": 1059025920.0, + "85": 1059004416.0, + "86": 1059015680.0, + "87": 1058998272.0, + "88": 1058990080.0, "89": 1059011584.0, - "90": 1059018752.0, - "91": 1058996736.0, - "92": 1059015680.0, - "93": 1058995712.0, - "94": 1058995200.0, - "95": 1059003392.0, - "96": 1059005440.0, - "97": 1059012096.0, - "98": 1059019264.0, - "99": 1058999808.0, - "100": 1058989568.0 + "90": 1059017216.0, + "91": 1058998272.0, + "92": 1059014656.0, + "93": 1058994176.0, + "94": 1058994176.0, + "95": 1059005952.0, + "96": 1059005952.0, + "97": 1059014144.0, + "98": 1059020288.0, + "99": 1059002368.0, + "100": 1058991104.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3362050048.0, - "2": 3674120704.0, - "3": 3674120704.0, - "4": 3674120704.0, - "5": 3674120704.0, - "6": 3674120704.0, - "7": 3674120704.0, - "8": 3674120704.0, - "9": 3674120704.0, - "10": 3674120704.0, - "11": 3674120704.0, - "12": 3674120704.0, - "13": 3674120704.0, - "14": 3674120704.0, - "15": 3674120704.0, - "16": 3674120704.0, - "17": 3674120704.0, - "18": 3674120704.0, - "19": 3674120704.0, - "20": 3674120704.0, - "21": 3674120704.0, - "22": 3674120704.0, - "23": 3674120704.0, - "24": 3674120704.0, - "25": 3674120704.0, - "26": 3674120704.0, - "27": 3674120704.0, - "28": 3674120704.0, - "29": 3674120704.0, - "30": 3674120704.0, - "31": 3674120704.0, - "32": 3674120704.0, - "33": 3674120704.0, - "34": 3674120704.0, - "35": 3674120704.0, - "36": 3674120704.0, - "37": 3674120704.0, - "38": 3674120704.0, - "39": 3674120704.0, - "40": 3674120704.0, - "41": 3674120704.0, - "42": 3674120704.0, - "43": 3674120704.0, - "44": 3674120704.0, - "45": 3674120704.0, - "46": 3674120704.0, - "47": 3674120704.0, - "48": 3674120704.0, - "49": 3674120704.0, - "50": 3674120704.0, - "51": 3674120704.0, - "52": 3674120704.0, - "53": 3674120704.0, - "54": 3674120704.0, - "55": 3674120704.0, - "56": 3674120704.0, - "57": 3674120704.0, - "58": 3674120704.0, - "59": 3674120704.0, - "60": 3674120704.0, - "61": 3674120704.0, - "62": 3674120704.0, - "63": 3674120704.0, - "64": 3674120704.0, - "65": 3674120704.0, - "66": 3674120704.0, - "67": 3674120704.0, - "68": 3674120704.0, - "69": 3674120704.0, - "70": 3674120704.0, - "71": 3674120704.0, - "72": 3674120704.0, - "73": 3674120704.0, - "74": 3674120704.0, - "75": 3674120704.0, - "76": 3674120704.0, - "77": 3674120704.0, - "78": 3674120704.0, - "79": 3674120704.0, - "80": 3674120704.0, - "81": 3674120704.0, - "82": 3674120704.0, - "83": 3674120704.0, - "84": 3674120704.0, - "85": 3674120704.0, - "86": 3674120704.0, - "87": 3674120704.0, - "88": 3674120704.0, - "89": 3674120704.0, - "90": 3674120704.0, - "91": 3674120704.0, - "92": 3674120704.0, - "93": 3674120704.0, - "94": 3674120704.0, - "95": 3674120704.0, - "96": 3674120704.0, - "97": 3674120704.0, - "98": 3674120704.0, - "99": 3674120704.0, - "100": 3674120704.0 + "1": 3362376192.0, + "2": 3675800064.0, + "3": 3675800064.0, + "4": 3675800064.0, + "5": 3675800064.0, + "6": 3675800064.0, + "7": 3675800064.0, + "8": 3675800064.0, + "9": 3675800064.0, + "10": 3675800064.0, + "11": 3675800064.0, + "12": 3675800064.0, + "13": 3675800064.0, + "14": 3675800064.0, + "15": 3675800064.0, + "16": 3675800064.0, + "17": 3675800064.0, + "18": 3675800064.0, + "19": 3675800064.0, + "20": 3675800064.0, + "21": 3675800064.0, + "22": 3675800064.0, + "23": 3675800064.0, + "24": 3675800064.0, + "25": 3675800064.0, + "26": 3675800064.0, + "27": 3675800064.0, + "28": 3675800064.0, + "29": 3675800064.0, + "30": 3675800064.0, + "31": 3675800064.0, + "32": 3675800064.0, + "33": 3675800064.0, + "34": 3675800064.0, + "35": 3675800064.0, + "36": 3675800064.0, + "37": 3675800064.0, + "38": 3675800064.0, + "39": 3675800064.0, + "40": 3675800064.0, + "41": 3675800064.0, + "42": 3675800064.0, + "43": 3675800064.0, + "44": 3675800064.0, + "45": 3675800064.0, + "46": 3675800064.0, + "47": 3675800064.0, + "48": 3675800064.0, + "49": 3675800064.0, + "50": 3675800064.0, + "51": 3675800064.0, + "52": 3675800064.0, + "53": 3675800064.0, + "54": 3675800064.0, + "55": 3675800064.0, + "56": 3675800064.0, + "57": 3675800064.0, + "58": 3675800064.0, + "59": 3675800064.0, + "60": 3675800064.0, + "61": 3675800064.0, + "62": 3675800064.0, + "63": 3675800064.0, + "64": 3675800064.0, + "65": 3675800064.0, + "66": 3675800064.0, + "67": 3675800064.0, + "68": 3675800064.0, + "69": 3675800064.0, + "70": 3675800064.0, + "71": 3675800064.0, + "72": 3675800064.0, + "73": 3675800064.0, + "74": 3675800064.0, + "75": 3675800064.0, + "76": 3675800064.0, + "77": 3675800064.0, + "78": 3675800064.0, + "79": 3675800064.0, + "80": 3675800064.0, + "81": 3675800064.0, + "82": 3675800064.0, + "83": 3675800064.0, + "84": 3675800064.0, + "85": 3675800064.0, + "86": 3675800064.0, + "87": 3675800064.0, + "88": 3675800064.0, + "89": 3675800064.0, + "90": 3675800064.0, + "91": 3675800064.0, + "92": 3675800064.0, + "93": 3675800064.0, + "94": 3675800064.0, + "95": 3675800064.0, + "96": 3675800064.0, + "97": 3675800064.0, + "98": 3675800064.0, + "99": 3675800064.0, + "100": 3675800064.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.35509, - "3": 0.3679, - "4": 0.4669, - "5": 0.45112, - "6": 0.56785, - "7": 0.92038, - "8": 1.0716, - "9": 0.68718, - "10": 0.37326, - "11": 1.22305, - "12": 0.35209, - "13": 0.70298, - "14": 0.77455, - "15": 0.38536, - "16": 0.85355, - "17": 0.60318, - "18": 1.00323, - "19": 0.32263, - "20": 0.53113, - "21": 0.67641, - "22": 0.63731, - "23": 0.74937, - "24": 0.32413, - "25": 0.53498, - "26": 1.10121, - "27": 0.33237, - "28": 0.54695, - "29": 0.86874, - "30": 0.32495, - "31": 0.31728, - "32": 1.03664, - "33": 0.57349, - "34": 0.32496, - "35": 0.72146, - "36": 0.60269, - "37": 0.59376, - "38": 0.43416, - "39": 0.64967, - "40": 0.79935, - "41": 0.56236, - "42": 0.69708, - "43": 0.51618, - "44": 0.9736, - "45": 0.51171, - "46": 0.77559, - "47": 0.62083, - "48": 0.9472, - "49": 0.5105, - "50": 0.42938, - "51": 0.42508, - "52": 0.37932, - "53": 0.43119, - "54": 0.8941, - "55": 0.51209, - "56": 0.83069, - "57": 0.43623, - "58": 0.35817, - "59": 0.81733, - "60": 0.63479, - "61": 0.31775, - "62": 0.72748, - "63": 0.58134, - "64": 0.91265, - "65": 1.01681, - "66": 0.99293, - "67": 0.73101, - "68": 0.56925, - "69": 1.43381, - "70": 1.08257, - "71": 0.31943, - "72": 0.61016, - "73": 1.05691, - "74": 0.51477, - "75": 1.5774, - "76": 0.44788, - "77": 0.54714, - "78": 1.57928, - "79": 0.49848, - "80": 0.63411, - "81": 0.67121, - "82": 0.72404, - "83": 0.56718, - "84": 0.78362, - "85": 0.31149, - "86": 0.45683, - "87": 1.58911, - "88": 0.53302, - "89": 0.53568, - "90": 0.54116, - "91": 0.57453, - "92": 1.05594, - "93": 0.32571, - "94": 0.68233, - "95": 0.68219, - "96": 0.65202, - "97": 0.35559, - "98": 0.4391, - "99": 1.42006, - "100": 0.33472 + "2": 6.2705, + "3": 0.32934, + "4": 0.37786, + "5": 0.35761, + "6": 0.54321, + "7": 0.76253, + "8": 1.0029, + "9": 0.62789, + "10": 0.3548, + "11": 1.21255, + "12": 0.27873, + "13": 0.57677, + "14": 0.7125, + "15": 0.39181, + "16": 0.92771, + "17": 0.58559, + "18": 0.82565, + "19": 0.31867, + "20": 0.59221, + "21": 0.81858, + "22": 0.63029, + "23": 0.66412, + "24": 0.30944, + "25": 0.54153, + "26": 1.07474, + "27": 0.27934, + "28": 0.59109, + "29": 0.71648, + "30": 0.28695, + "31": 0.28344, + "32": 1.07691, + "33": 0.51232, + "34": 0.57504, + "35": 0.28378, + "36": 0.56943, + "37": 0.49778, + "38": 0.39966, + "39": 0.5387, + "40": 0.53487, + "41": 0.51279, + "42": 0.67421, + "43": 0.49433, + "44": 0.93007, + "45": 0.4214, + "46": 0.64116, + "47": 0.56769, + "48": 0.76864, + "49": 0.38162, + "50": 0.59715, + "51": 0.35905, + "52": 0.37257, + "53": 0.43931, + "54": 0.62524, + "55": 0.5538, + "56": 0.81516, + "57": 0.41221, + "58": 0.5429, + "59": 0.62802, + "60": 0.70209, + "61": 0.29698, + "62": 0.822, + "63": 0.46607, + "64": 0.71079, + "65": 0.48871, + "66": 0.64075, + "67": 0.55829, + "68": 0.28843, + "69": 0.74298, + "70": 1.04221, + "71": 0.2812, + "72": 0.62599, + "73": 0.59949, + "74": 0.44705, + "75": 0.93853, + "76": 0.43742, + "77": 0.44632, + "78": 1.3474, + "79": 0.36346, + "80": 0.62457, + "81": 0.63583, + "82": 0.53485, + "83": 0.59852, + "84": 0.7293, + "85": 0.29224, + "86": 0.28485, + "87": 1.02179, + "88": 0.7876, + "89": 0.58867, + "90": 0.37329, + "91": 0.48666, + "92": 0.69345, + "93": 0.27982, + "94": 0.7062, + "95": 0.8814, + "96": 0.37726, + "97": 0.6088, + "98": 0.56104, + "99": 1.29227, + "100": 0.32749 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json index 709dea2488b..008353905e2 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90652, - "4": 10.90542, - "5": 10.91347, - "6": 10.89722, - "7": 10.89815, - "8": 10.91325, - "9": 10.89596, - "10": 10.90411, - "11": 10.9007, - "12": 10.91797, - "13": 10.88426, - "14": 10.87521, - "15": 10.90976, - "16": 10.90312, - "17": 10.88662, - "18": 10.89386, - "19": 10.89448, - "20": 10.89126, - "21": 10.88678, - "22": 10.90299, - "23": 10.91234, - "24": 10.88568, - "25": 10.90086, - "26": 10.88913, - "27": 10.89694, - "28": 10.88118, - "29": 10.88993, - "30": 10.9105, - "31": 10.89106, - "32": 10.88858, - "33": 10.8978, - "34": 10.87329, - "35": 10.89659, - "36": 10.90752, - "37": 10.871, - "38": 10.87897, - "39": 10.88601, - "40": 10.89213, - "41": 10.88357, - "42": 10.89522, - "43": 10.88028, - "44": 10.88496, - "45": 10.88315, - "46": 10.88638, - "47": 10.88589, - "48": 10.86391, - "49": 10.87629, - "50": 10.8831, - "51": 10.89327, - "52": 10.8706, - "53": 10.8583, - "54": 10.87089, - "55": 10.8681, - "56": 10.87353, - "57": 10.84429, - "58": 10.85622, - "59": 10.84583, - "60": 10.84335, - "61": 10.86109, - "62": 10.85723, - "63": 10.86005, - "64": 10.83881, - "65": 10.82773, - "66": 10.84549, - "67": 10.82785, - "68": 10.82933, - "69": 10.81858, - "70": 10.82823, - "71": 10.82685, - "72": 10.80812, - "73": 10.8077, - "74": 10.80648, - "75": 10.81481, - "76": 10.81136, - "77": 10.80743, - "78": 10.79191, - "79": 10.80241, - "80": 10.78881, - "81": 10.79504, - "82": 10.79535, - "83": 10.78597, - "84": 10.7569, - "85": 10.76252, + "1": 10.90111, + "2": 10.8957, + "3": 10.90636, + "4": 10.90618, + "5": 10.91363, + "6": 10.89761, + "7": 10.89776, + "8": 10.91415, + "9": 10.89561, + "10": 10.9037, + "11": 10.90078, + "12": 10.91896, + "13": 10.88436, + "14": 10.8755, + "15": 10.91016, + "16": 10.90356, + "17": 10.88647, + "18": 10.89373, + "19": 10.89488, + "20": 10.89145, + "21": 10.8867, + "22": 10.90339, + "23": 10.9122, + "24": 10.88545, + "25": 10.90147, + "26": 10.88807, + "27": 10.89667, + "28": 10.88181, + "29": 10.88999, + "30": 10.91088, + "31": 10.89164, + "32": 10.88874, + "33": 10.89699, + "34": 10.87341, + "35": 10.89671, + "36": 10.90754, + "37": 10.87082, + "38": 10.87845, + "39": 10.8861, + "40": 10.89256, + "41": 10.88453, + "42": 10.89553, + "43": 10.88019, + "44": 10.88447, + "45": 10.88361, + "46": 10.88675, + "47": 10.88597, + "48": 10.8638, + "49": 10.8766, + "50": 10.88241, + "51": 10.89271, + "52": 10.87137, + "53": 10.85905, + "54": 10.87009, + "55": 10.86786, + "56": 10.87291, + "57": 10.84458, + "58": 10.85672, + "59": 10.84682, + "60": 10.84354, + "61": 10.86079, + "62": 10.85703, + "63": 10.86049, + "64": 10.83806, + "65": 10.82753, + "66": 10.84631, + "67": 10.82788, + "68": 10.83033, + "69": 10.81867, + "70": 10.82807, + "71": 10.82687, + "72": 10.80846, + "73": 10.80745, + "74": 10.8061, + "75": 10.81421, + "76": 10.81095, + "77": 10.80817, + "78": 10.79232, + "79": 10.80249, + "80": 10.78847, + "81": 10.79536, + "82": 10.79574, + "83": 10.78549, + "84": 10.75707, + "85": 10.76214, "86": 10.77729, - "87": 10.79571, - "88": 10.77474, - "89": 10.77531, - "90": 10.76553, - "91": 10.74039, - "92": 10.76056, - "93": 10.74732, - "94": 10.73435, - "95": 10.75251, - "96": 10.72334, - "97": 10.71517, - "98": 10.72635, - "99": 10.74623, - "100": 10.69589 + "87": 10.79586, + "88": 10.77534, + "89": 10.77602, + "90": 10.76492, + "91": 10.74145, + "92": 10.75946, + "93": 10.74599, + "94": 10.73392, + "95": 10.75202, + "96": 10.72408, + "97": 10.71554, + "98": 10.7261, + "99": 10.74663, + "100": 10.69609 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1369.0, - "4": 1274.0, - "5": 1227.0, - "6": 1240.0, - "7": 1282.0, - "8": 1206.0, - "9": 1161.0, - "10": 1142.0, - "11": 1186.0, - "12": 1297.0, + "1": 1133.0, + "2": 1120.0, + "3": 1360.0, + "4": 1289.0, + "5": 1206.0, + "6": 1218.0, + "7": 1344.0, + "8": 1181.0, + "9": 1206.0, + "10": 1139.0, + "11": 1252.0, + "12": 1218.0, "13": 1288.0, - "14": 1215.0, - "15": 1218.0, - "16": 1231.0, - "17": 1213.0, - "18": 1105.0, - "19": 1178.0, - "20": 1096.0, - "21": 1231.0, - "22": 1266.0, - "23": 1278.0, - "24": 1169.0, - "25": 1196.0, - "26": 1212.0, - "27": 1344.0, - "28": 1218.0, - "29": 1272.0, - "30": 1288.0, - "31": 1188.0, - "32": 1250.0, - "33": 1168.0, - "34": 1259.0, - "35": 1296.0, - "36": 1204.0, - "37": 1184.0, - "38": 1219.0, - "39": 1386.0, - "40": 1284.0, - "41": 1298.0, - "42": 1146.0, - "43": 1156.0, - "44": 1277.0, - "45": 1128.0, - "46": 1381.0, - "47": 1194.0, - "48": 1137.0, - "49": 1369.0, - "50": 1277.0, - "51": 1257.0, - "52": 1159.0, - "53": 1253.0, - "54": 1202.0, - "55": 1181.0, - "56": 1178.0, - "57": 1233.0, - "58": 1260.0, - "59": 1084.0, - "60": 1188.0, - "61": 1240.0, - "62": 1173.0, - "63": 1188.0, - "64": 1248.0, - "65": 1222.0, - "66": 1194.0, - "67": 1269.0, - "68": 1288.0, - "69": 1161.0, - "70": 1300.0, - "71": 1235.0, - "72": 1206.0, - "73": 1272.0, - "74": 1254.0, - "75": 1392.0, - "76": 1292.0, - "77": 1155.0, - "78": 1245.0, - "79": 1316.0, - "80": 1145.0, - "81": 1304.0, - "82": 1213.0, - "83": 1290.0, - "84": 1121.0, - "85": 1162.0, - "86": 1310.0, - "87": 1123.0, - "88": 1197.0, - "89": 1089.0, - "90": 1438.0, - "91": 1300.0, - "92": 1143.0, - "93": 1293.0, - "94": 1353.0, - "95": 1061.0, - "96": 1250.0, - "97": 1216.0, - "98": 1310.0, - "99": 1121.0, - "100": 1167.0 + "14": 1162.0, + "15": 1248.0, + "16": 1265.0, + "17": 1188.0, + "18": 1115.0, + "19": 1172.0, + "20": 1089.0, + "21": 1213.0, + "22": 1244.0, + "23": 1290.0, + "24": 1071.0, + "25": 1243.0, + "26": 1216.0, + "27": 1393.0, + "28": 1179.0, + "29": 1194.0, + "30": 1353.0, + "31": 1240.0, + "32": 1343.0, + "33": 1190.0, + "34": 1196.0, + "35": 1270.0, + "36": 1169.0, + "37": 1196.0, + "38": 1212.0, + "39": 1344.0, + "40": 1272.0, + "41": 1350.0, + "42": 1153.0, + "43": 1231.0, + "44": 1234.0, + "45": 1094.0, + "46": 1458.0, + "47": 1192.0, + "48": 1140.0, + "49": 1412.0, + "50": 1316.0, + "51": 1261.0, + "52": 1161.0, + "53": 1242.0, + "54": 1183.0, + "55": 1246.0, + "56": 1243.0, + "57": 1208.0, + "58": 1283.0, + "59": 1108.0, + "60": 1224.0, + "61": 1277.0, + "62": 1192.0, + "63": 1234.0, + "64": 1285.0, + "65": 1166.0, + "66": 1238.0, + "67": 1128.0, + "68": 1281.0, + "69": 1115.0, + "70": 1255.0, + "71": 1199.0, + "72": 1239.0, + "73": 1264.0, + "74": 1318.0, + "75": 1344.0, + "76": 1250.0, + "77": 1197.0, + "78": 1241.0, + "79": 1270.0, + "80": 1227.0, + "81": 1238.0, + "82": 1078.0, + "83": 1220.0, + "84": 1190.0, + "85": 1187.0, + "86": 1238.0, + "87": 1074.0, + "88": 1182.0, + "89": 1184.0, + "90": 1343.0, + "91": 1308.0, + "92": 1173.0, + "93": 1342.0, + "94": 1400.0, + "95": 1059.0, + "96": 1129.0, + "97": 1263.0, + "98": 1344.0, + "99": 1065.0, + "100": 1185.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1096817664.0, - "2": 1096807936.0, - "3": 1096783872.0, - "4": 1096764416.0, - "5": 1096815104.0, - "6": 1096799744.0, - "7": 1096722944.0, - "8": 1096760832.0, - "9": 1096762880.0, - "10": 1096796160.0, - "11": 1096770048.0, - "12": 1096778752.0, - "13": 1096783360.0, - "14": 1096783360.0, - "15": 1096762880.0, - "16": 1096748032.0, - "17": 1096756224.0, - "18": 1096758272.0, - "19": 1096789504.0, - "20": 1096754176.0, - "21": 1096804352.0, - "22": 1096830464.0, - "23": 1096802304.0, - "24": 1096719872.0, - "25": 1096800768.0, - "26": 1096830464.0, - "27": 1096781824.0, - "28": 1096736768.0, - "29": 1096810496.0, - "30": 1096745984.0, - "31": 1096750592.0, - "32": 1096758784.0, - "33": 1096764928.0, - "34": 1096742400.0, - "35": 1096730112.0, - "36": 1096793600.0, - "37": 1096743424.0, - "38": 1096801792.0, - "39": 1096810496.0, - "40": 1096754688.0, - "41": 1096777728.0, - "42": 1096720384.0, - "43": 1096785920.0, - "44": 1096770560.0, - "45": 1096745472.0, - "46": 1096808960.0, - "47": 1096766464.0, - "48": 1096794112.0, - "49": 1096755712.0, - "50": 1096802816.0, - "51": 1096785920.0, - "52": 1096758784.0, - "53": 1096707072.0, - "54": 1096805888.0, - "55": 1096795648.0, - "56": 1096794112.0, - "57": 1096855552.0, - "58": 1096790528.0, - "59": 1096795648.0, - "60": 1096774144.0, - "61": 1096743936.0, - "62": 1096733696.0, - "63": 1096740352.0, - "64": 1096771584.0, - "65": 1096778752.0, - "66": 1096792064.0, - "67": 1096775168.0, - "68": 1096774144.0, - "69": 1096738816.0, - "70": 1096822784.0, - "71": 1096799744.0, - "72": 1096799232.0, - "73": 1096731136.0, - "74": 1096769536.0, - "75": 1096816640.0, - "76": 1096772096.0, - "77": 1096817664.0, - "78": 1096759296.0, - "79": 1096826880.0, - "80": 1096746496.0, - "81": 1096800768.0, - "82": 1096747520.0, - "83": 1096810496.0, - "84": 1096782336.0, - "85": 1096786432.0, - "86": 1096771072.0, - "87": 1096785408.0, - "88": 1096768512.0, - "89": 1096757760.0, - "90": 1096779776.0, - "91": 1096765440.0, - "92": 1096763392.0, - "93": 1096793600.0, - "94": 1096767488.0, - "95": 1096791552.0, - "96": 1096782848.0, - "97": 1096751104.0, - "98": 1096788480.0, - "99": 1096806400.0, - "100": 1096837120.0 + "1": 1096719872.0, + "2": 1096710656.0, + "3": 1096685568.0, + "4": 1096666112.0, + "5": 1096716288.0, + "6": 1096698368.0, + "7": 1096625664.0, + "8": 1096664064.0, + "9": 1096666624.0, + "10": 1096697344.0, + "11": 1096671744.0, + "12": 1096680448.0, + "13": 1096684032.0, + "14": 1096687104.0, + "15": 1096663040.0, + "16": 1096648192.0, + "17": 1096657920.0, + "18": 1096658944.0, + "19": 1096689152.0, + "20": 1096654336.0, + "21": 1096707584.0, + "22": 1096734208.0, + "23": 1096705024.0, + "24": 1096621568.0, + "25": 1096701952.0, + "26": 1096734208.0, + "27": 1096685056.0, + "28": 1096639488.0, + "29": 1096713216.0, + "30": 1096646656.0, + "31": 1096652800.0, + "32": 1096659456.0, + "33": 1096667648.0, + "34": 1096645120.0, + "35": 1096631296.0, + "36": 1096694272.0, + "37": 1096644096.0, + "38": 1096703488.0, + "39": 1096710144.0, + "40": 1096657408.0, + "41": 1096678912.0, + "42": 1096621568.0, + "43": 1096688128.0, + "44": 1096673792.0, + "45": 1096646656.0, + "46": 1096709632.0, + "47": 1096666624.0, + "48": 1096695808.0, + "49": 1096656896.0, + "50": 1096704000.0, + "51": 1096687616.0, + "52": 1096661504.0, + "53": 1096610816.0, + "54": 1096705024.0, + "55": 1096696320.0, + "56": 1096693248.0, + "57": 1096753664.0, + "58": 1096691712.0, + "59": 1096695808.0, + "60": 1096675840.0, + "61": 1096647168.0, + "62": 1096636416.0, + "63": 1096645632.0, + "64": 1096672256.0, + "65": 1096679936.0, + "66": 1096693248.0, + "67": 1096674816.0, + "68": 1096677376.0, + "69": 1096642560.0, + "70": 1096725504.0, + "71": 1096706560.0, + "72": 1096703488.0, + "73": 1096632320.0, + "74": 1096673792.0, + "75": 1096717312.0, + "76": 1096673280.0, + "77": 1096714752.0, + "78": 1096664064.0, + "79": 1096729600.0, + "80": 1096649216.0, + "81": 1096703488.0, + "82": 1096649728.0, + "83": 1096710144.0, + "84": 1096683008.0, + "85": 1096688640.0, + "86": 1096672256.0, + "87": 1096687104.0, + "88": 1096668672.0, + "89": 1096658944.0, + "90": 1096683008.0, + "91": 1096666112.0, + "92": 1096664576.0, + "93": 1096695808.0, + "94": 1096671744.0, + "95": 1096692736.0, + "96": 1096684544.0, + "97": 1096652800.0, + "98": 1096691200.0, + "99": 1096708608.0, + "100": 1096738304.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3258557952.0, - "2": 3590636544.0, - "3": 3590636544.0, - "4": 3590636544.0, - "5": 3596905472.0, - "6": 3596905472.0, - "7": 3596905472.0, - "8": 3596905472.0, - "9": 3596905472.0, - "10": 3596905472.0, - "11": 3596905472.0, - "12": 3596905472.0, - "13": 3596905472.0, - "14": 3596905472.0, - "15": 3596905472.0, - "16": 3596905472.0, - "17": 3596905472.0, - "18": 3596905472.0, - "19": 3596905472.0, - "20": 3596905472.0, - "21": 3596905472.0, - "22": 3609437184.0, - "23": 3609437184.0, - "24": 3609437184.0, - "25": 3609437184.0, - "26": 3611321344.0, - "27": 3611321344.0, - "28": 3611321344.0, - "29": 3611321344.0, - "30": 3611321344.0, - "31": 3611321344.0, - "32": 3611321344.0, - "33": 3611321344.0, - "34": 3611321344.0, - "35": 3611321344.0, - "36": 3611321344.0, - "37": 3611321344.0, - "38": 3611321344.0, - "39": 3611321344.0, - "40": 3611321344.0, - "41": 3611321344.0, - "42": 3611321344.0, - "43": 3611321344.0, - "44": 3611321344.0, - "45": 3611321344.0, - "46": 3611321344.0, - "47": 3611321344.0, - "48": 3611321344.0, - "49": 3611321344.0, - "50": 3611321344.0, - "51": 3611321344.0, - "52": 3611321344.0, - "53": 3611321344.0, - "54": 3611321344.0, - "55": 3611321344.0, - "56": 3611321344.0, - "57": 3630413312.0, - "58": 3630413312.0, - "59": 3630413312.0, - "60": 3630413312.0, - "61": 3630413312.0, - "62": 3630413312.0, - "63": 3630413312.0, - "64": 3630413312.0, - "65": 3630413312.0, - "66": 3630413312.0, - "67": 3630413312.0, - "68": 3630413312.0, - "69": 3630413312.0, - "70": 3630413312.0, - "71": 3630413312.0, - "72": 3630413312.0, - "73": 3630413312.0, - "74": 3630413312.0, - "75": 3630413312.0, - "76": 3630413312.0, - "77": 3630413312.0, - "78": 3630413312.0, - "79": 3630413312.0, - "80": 3630413312.0, - "81": 3630413312.0, - "82": 3630413312.0, - "83": 3630413312.0, - "84": 3630413312.0, - "85": 3630413312.0, - "86": 3630413312.0, - "87": 3630413312.0, - "88": 3630413312.0, - "89": 3630413312.0, - "90": 3630413312.0, - "91": 3630413312.0, - "92": 3630413312.0, - "93": 3630413312.0, - "94": 3630413312.0, - "95": 3630413312.0, - "96": 3630413312.0, - "97": 3630413312.0, - "98": 3630413312.0, - "99": 3630413312.0, - "100": 3630413312.0 + "1": 3257109504.0, + "2": 3590822912.0, + "3": 3590822912.0, + "4": 3590822912.0, + "5": 3596996096.0, + "6": 3596996096.0, + "7": 3596996096.0, + "8": 3596996096.0, + "9": 3596996096.0, + "10": 3596996096.0, + "11": 3596996096.0, + "12": 3596996096.0, + "13": 3596996096.0, + "14": 3596996096.0, + "15": 3596996096.0, + "16": 3596996096.0, + "17": 3596996096.0, + "18": 3596996096.0, + "19": 3596996096.0, + "20": 3596996096.0, + "21": 3596996096.0, + "22": 3610441728.0, + "23": 3610441728.0, + "24": 3610441728.0, + "25": 3610441728.0, + "26": 3611658240.0, + "27": 3611658240.0, + "28": 3611658240.0, + "29": 3611658240.0, + "30": 3611658240.0, + "31": 3611658240.0, + "32": 3611658240.0, + "33": 3611658240.0, + "34": 3611658240.0, + "35": 3611658240.0, + "36": 3611658240.0, + "37": 3611658240.0, + "38": 3611658240.0, + "39": 3611658240.0, + "40": 3611658240.0, + "41": 3611658240.0, + "42": 3611658240.0, + "43": 3611658240.0, + "44": 3611658240.0, + "45": 3611658240.0, + "46": 3611658240.0, + "47": 3611658240.0, + "48": 3611658240.0, + "49": 3611658240.0, + "50": 3611658240.0, + "51": 3611658240.0, + "52": 3611658240.0, + "53": 3611658240.0, + "54": 3611658240.0, + "55": 3611658240.0, + "56": 3611658240.0, + "57": 3625479168.0, + "58": 3625479168.0, + "59": 3625479168.0, + "60": 3625479168.0, + "61": 3625479168.0, + "62": 3625479168.0, + "63": 3625479168.0, + "64": 3625479168.0, + "65": 3625479168.0, + "66": 3625479168.0, + "67": 3625479168.0, + "68": 3625479168.0, + "69": 3625479168.0, + "70": 3625479168.0, + "71": 3625479168.0, + "72": 3625479168.0, + "73": 3625479168.0, + "74": 3625479168.0, + "75": 3625479168.0, + "76": 3625479168.0, + "77": 3625479168.0, + "78": 3625479168.0, + "79": 3625479168.0, + "80": 3625479168.0, + "81": 3625479168.0, + "82": 3625479168.0, + "83": 3625479168.0, + "84": 3625479168.0, + "85": 3625479168.0, + "86": 3625479168.0, + "87": 3625479168.0, + "88": 3625479168.0, + "89": 3625479168.0, + "90": 3625479168.0, + "91": 3625479168.0, + "92": 3625479168.0, + "93": 3625479168.0, + "94": 3625479168.0, + "95": 3625479168.0, + "96": 3625479168.0, + "97": 3625479168.0, + "98": 3625479168.0, + "99": 3625479168.0, + "100": 3625479168.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.78938, - "3": 0.60373, - "4": 0.59311, - "5": 0.59195, - "6": 0.59273, - "7": 0.57837, - "8": 0.60657, - "9": 0.57308, - "10": 0.57962, - "11": 0.59121, - "12": 0.57012, - "13": 0.57956, - "14": 0.57252, - "15": 0.5778, - "16": 0.57556, - "17": 0.58331, - "18": 0.56468, - "19": 0.57301, - "20": 0.56966, - "21": 0.56425, - "22": 0.57413, - "23": 0.55995, - "24": 0.57786, - "25": 0.57348, - "26": 0.56472, - "27": 0.55511, - "28": 0.56315, - "29": 0.57013, - "30": 0.56578, - "31": 0.57204, - "32": 0.56786, - "33": 0.56996, - "34": 0.56576, - "35": 0.56723, - "36": 0.56311, - "37": 0.55743, - "38": 0.56454, - "39": 0.56626, - "40": 0.56817, - "41": 0.56308, - "42": 0.56431, - "43": 0.55892, - "44": 0.56373, - "45": 0.57067, - "46": 0.55855, - "47": 0.56537, - "48": 0.56161, - "49": 0.55823, - "50": 0.55897, - "51": 1.51215, - "52": 0.58969, - "53": 0.5794, - "54": 0.56179, - "55": 0.55753, - "56": 0.55763, - "57": 0.56517, - "58": 0.56733, - "59": 0.55665, - "60": 0.5759, - "61": 0.65482, - "62": 0.57041, - "63": 0.55863, - "64": 0.56279, - "65": 0.55663, - "66": 0.56047, - "67": 0.5619, - "68": 0.56148, - "69": 0.55887, - "70": 0.5592, - "71": 0.56258, - "72": 0.56253, - "73": 0.56437, - "74": 0.56609, - "75": 0.55638, - "76": 0.57034, - "77": 0.56328, - "78": 0.56443, - "79": 0.56674, - "80": 0.56048, - "81": 0.5663, - "82": 0.55975, - "83": 0.55919, - "84": 0.55883, - "85": 0.55711, - "86": 0.56565, - "87": 0.55683, - "88": 0.58115, - "89": 0.57514, - "90": 0.57865, - "91": 0.56708, - "92": 0.56269, - "93": 0.55817, - "94": 0.56214, - "95": 0.56258, - "96": 0.56495, - "97": 0.56018, - "98": 0.55666, - "99": 0.5647, - "100": 0.55567 + "2": 4.86849, + "3": 0.61494, + "4": 0.5965, + "5": 0.61106, + "6": 0.5937, + "7": 0.59275, + "8": 0.6049, + "9": 0.59319, + "10": 0.5813, + "11": 0.59548, + "12": 0.58644, + "13": 0.58842, + "14": 0.58678, + "15": 0.58232, + "16": 0.57827, + "17": 0.59409, + "18": 0.57001, + "19": 0.5811, + "20": 0.58807, + "21": 0.57391, + "22": 0.58056, + "23": 0.57849, + "24": 0.57957, + "25": 0.58076, + "26": 0.57697, + "27": 0.57129, + "28": 0.56963, + "29": 0.57313, + "30": 0.58722, + "31": 0.56864, + "32": 0.58093, + "33": 0.57098, + "34": 0.57288, + "35": 0.56666, + "36": 0.57178, + "37": 0.57491, + "38": 0.57664, + "39": 0.57238, + "40": 0.57818, + "41": 0.57, + "42": 0.57122, + "43": 0.57585, + "44": 0.56965, + "45": 0.57757, + "46": 0.5697, + "47": 0.5715, + "48": 0.56917, + "49": 0.56689, + "50": 0.5666, + "51": 0.61837, + "52": 0.60013, + "53": 0.57942, + "54": 0.57481, + "55": 0.5659, + "56": 0.57608, + "57": 0.5733, + "58": 0.57193, + "59": 0.57003, + "60": 0.57483, + "61": 0.58014, + "62": 0.57703, + "63": 0.57581, + "64": 0.56581, + "65": 0.56386, + "66": 0.56868, + "67": 0.56779, + "68": 0.57195, + "69": 0.56991, + "70": 0.57527, + "71": 0.56705, + "72": 0.57502, + "73": 0.58215, + "74": 0.57174, + "75": 0.57579, + "76": 0.56477, + "77": 0.58212, + "78": 0.57888, + "79": 0.57306, + "80": 0.57423, + "81": 0.57569, + "82": 0.57635, + "83": 0.57094, + "84": 0.56828, + "85": 0.57265, + "86": 0.5791, + "87": 0.57807, + "88": 0.58044, + "89": 0.5686, + "90": 0.57306, + "91": 0.57247, + "92": 0.56836, + "93": 0.56635, + "94": 0.56939, + "95": 0.57188, + "96": 0.57753, + "97": 0.57604, + "98": 0.56375, + "99": 0.57477, + "100": 0.58271 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json index 467d686c187..14c7034d871 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.9252, - "4": 10.91732, - "5": 10.92716, - "6": 10.92029, - "7": 10.91689, - "8": 10.91796, - "9": 10.93832, - "10": 10.9236, - "11": 10.91613, - "12": 10.92602, - "13": 10.93472, - "14": 10.91527, - "15": 10.92824, - "16": 10.91861, - "17": 10.9271, - "18": 10.91296, - "19": 10.93023, - "20": 10.90016, - "21": 10.93966, - "22": 10.92562, - "23": 10.93028, - "24": 10.90543, - "25": 10.91525, - "26": 10.9253, - "27": 10.92741, - "28": 10.92037, - "29": 10.92588, + "1": 10.93064, + "2": 10.92264, + "3": 10.92453, + "4": 10.91706, + "5": 10.92766, + "6": 10.92069, + "7": 10.9166, + "8": 10.91808, + "9": 10.93792, + "10": 10.92356, + "11": 10.91567, + "12": 10.92593, + "13": 10.93441, + "14": 10.91528, + "15": 10.92879, + "16": 10.9189, + "17": 10.92699, + "18": 10.91334, + "19": 10.93068, + "20": 10.90002, + "21": 10.9404, + "22": 10.92589, + "23": 10.93025, + "24": 10.90535, + "25": 10.91492, + "26": 10.92587, + "27": 10.9279, + "28": 10.91992, + "29": 10.92593, "30": 10.91378, - "31": 10.91042, - "32": 10.91855, - "33": 10.91979, - "34": 10.90066, - "35": 10.91944, - "36": 10.90776, - "37": 10.91233, - "38": 10.92343, - "39": 10.91274, - "40": 10.91036, - "41": 10.91294, - "42": 10.90908, - "43": 10.90588, - "44": 10.90751, - "45": 10.89747, - "46": 10.90732, - "47": 10.90263, - "48": 10.88416, - "49": 10.90034, - "50": 10.89852, - "51": 10.89775, - "52": 10.9041, - "53": 10.90148, - "54": 10.88746, - "55": 10.88695, - "56": 10.88936, - "57": 10.88718, - "58": 10.88857, - "59": 10.88171, - "60": 10.87361, - "61": 10.8735, - "62": 10.86885, - "63": 10.87575, - "64": 10.85989, - "65": 10.86121, - "66": 10.85651, - "67": 10.86113, - "68": 10.86021, - "69": 10.85001, - "70": 10.86693, - "71": 10.84848, - "72": 10.84069, - "73": 10.84796, - "74": 10.83901, - "75": 10.83614, - "76": 10.83454, - "77": 10.82879, - "78": 10.82267, - "79": 10.82851, - "80": 10.82921, - "81": 10.81793, - "82": 10.8169, - "83": 10.80474, - "84": 10.78276, - "85": 10.78362, - "86": 10.80088, - "87": 10.7953, - "88": 10.79578, - "89": 10.77732, - "90": 10.78292, - "91": 10.78991, - "92": 10.7824, - "93": 10.75549, - "94": 10.76106, - "95": 10.7731, - "96": 10.73578, - "97": 10.74142, - "98": 10.74708, - "99": 10.76219, - "100": 10.74211 + "31": 10.9109, + "32": 10.91882, + "33": 10.92004, + "34": 10.90075, + "35": 10.91879, + "36": 10.90792, + "37": 10.91301, + "38": 10.92359, + "39": 10.9122, + "40": 10.91099, + "41": 10.91319, + "42": 10.90912, + "43": 10.90596, + "44": 10.90688, + "45": 10.89722, + "46": 10.90681, + "47": 10.90148, + "48": 10.8843, + "49": 10.90016, + "50": 10.89916, + "51": 10.89786, + "52": 10.90405, + "53": 10.90069, + "54": 10.88764, + "55": 10.88607, + "56": 10.88893, + "57": 10.88739, + "58": 10.88856, + "59": 10.88187, + "60": 10.87372, + "61": 10.87299, + "62": 10.86808, + "63": 10.87591, + "64": 10.85915, + "65": 10.86096, + "66": 10.85725, + "67": 10.86131, + "68": 10.86071, + "69": 10.85024, + "70": 10.86702, + "71": 10.84888, + "72": 10.83992, + "73": 10.84898, + "74": 10.839, + "75": 10.83634, + "76": 10.83445, + "77": 10.82965, + "78": 10.82317, + "79": 10.82842, + "80": 10.82931, + "81": 10.81726, + "82": 10.81715, + "83": 10.80494, + "84": 10.78359, + "85": 10.78354, + "86": 10.80074, + "87": 10.79543, + "88": 10.79476, + "89": 10.77699, + "90": 10.78295, + "91": 10.79036, + "92": 10.78271, + "93": 10.75468, + "94": 10.7609, + "95": 10.77286, + "96": 10.73628, + "97": 10.74115, + "98": 10.74693, + "99": 10.76285, + "100": 10.74247 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 49241.0, - "4": 48921.0, - "5": 50418.0, - "6": 49904.0, - "7": 51873.0, - "8": 50118.0, - "9": 50874.0, - "10": 52037.0, - "11": 50176.0, - "12": 48637.0, - "13": 52081.0, - "14": 50045.0, - "15": 48440.0, - "16": 49755.0, - "17": 49654.0, - "18": 51941.0, - "19": 51305.0, - "20": 52452.0, - "21": 50160.0, - "22": 49759.0, - "23": 53668.0, - "24": 51479.0, - "25": 48721.0, - "26": 52060.0, - "27": 51901.0, - "28": 48375.0, - "29": 50453.0, - "30": 51324.0, - "31": 52343.0, - "32": 49490.0, - "33": 51963.0, - "34": 50282.0, - "35": 51151.0, - "36": 51128.0, - "37": 49489.0, - "38": 50866.0, - "39": 50922.0, - "40": 50972.0, - "41": 48289.0, - "42": 50130.0, - "43": 51143.0, - "44": 47593.0, - "45": 54555.0, - "46": 48428.0, - "47": 50067.0, - "48": 50925.0, - "49": 54379.0, - "50": 51185.0, - "51": 48440.0, - "52": 50898.0, - "53": 49761.0, - "54": 48987.0, - "55": 49733.0, - "56": 48541.0, - "57": 48439.0, - "58": 52836.0, - "59": 50186.0, - "60": 49645.0, - "61": 47579.0, - "62": 50843.0, - "63": 48799.0, - "64": 56194.0, - "65": 51032.0, - "66": 49352.0, - "67": 52039.0, - "68": 49477.0, - "69": 54578.0, - "70": 49750.0, - "71": 49318.0, - "72": 52184.0, - "73": 53016.0, - "74": 49953.0, - "75": 50635.0, - "76": 51334.0, - "77": 50040.0, - "78": 49918.0, - "79": 48849.0, - "80": 51988.0, - "81": 50001.0, - "82": 46662.0, - "83": 52909.0, - "84": 48924.0, - "85": 50721.0, - "86": 49065.0, - "87": 46136.0, - "88": 48331.0, - "89": 48705.0, - "90": 52429.0, - "91": 50491.0, - "92": 50216.0, - "93": 50541.0, - "94": 51146.0, - "95": 48468.0, - "96": 49648.0, - "97": 49767.0, - "98": 48502.0, - "99": 49428.0, - "100": 47072.0 + "1": 49180.0, + "2": 50476.0, + "3": 49152.0, + "4": 49093.0, + "5": 49913.0, + "6": 49695.0, + "7": 51527.0, + "8": 49236.0, + "9": 50963.0, + "10": 51988.0, + "11": 50179.0, + "12": 48660.0, + "13": 52142.0, + "14": 50135.0, + "15": 47947.0, + "16": 49733.0, + "17": 49715.0, + "18": 51893.0, + "19": 50900.0, + "20": 52897.0, + "21": 50249.0, + "22": 49494.0, + "23": 53440.0, + "24": 51476.0, + "25": 48836.0, + "26": 52057.0, + "27": 51764.0, + "28": 48472.0, + "29": 50091.0, + "30": 51160.0, + "31": 52934.0, + "32": 49271.0, + "33": 52492.0, + "34": 49785.0, + "35": 51389.0, + "36": 50759.0, + "37": 49447.0, + "38": 50917.0, + "39": 50977.0, + "40": 51303.0, + "41": 47884.0, + "42": 50199.0, + "43": 51264.0, + "44": 48123.0, + "45": 54319.0, + "46": 47721.0, + "47": 50709.0, + "48": 50505.0, + "49": 54725.0, + "50": 50759.0, + "51": 48818.0, + "52": 50484.0, + "53": 49662.0, + "54": 49308.0, + "55": 49348.0, + "56": 48679.0, + "57": 48551.0, + "58": 52274.0, + "59": 49618.0, + "60": 49684.0, + "61": 47669.0, + "62": 50900.0, + "63": 48419.0, + "64": 55920.0, + "65": 51019.0, + "66": 49471.0, + "67": 51576.0, + "68": 49653.0, + "69": 54382.0, + "70": 50016.0, + "71": 49561.0, + "72": 51901.0, + "73": 52536.0, + "74": 49702.0, + "75": 50683.0, + "76": 51554.0, + "77": 49757.0, + "78": 49608.0, + "79": 49124.0, + "80": 52318.0, + "81": 49552.0, + "82": 46737.0, + "83": 52475.0, + "84": 48461.0, + "85": 50083.0, + "86": 49539.0, + "87": 46565.0, + "88": 48397.0, + "89": 48861.0, + "90": 52531.0, + "91": 50128.0, + "92": 50276.0, + "93": 50731.0, + "94": 51663.0, + "95": 48703.0, + "96": 49914.0, + "97": 49912.0, + "98": 48976.0, + "99": 48894.0, + "100": 47024.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1257367552.0, - "2": 1257389056.0, + "1": 1257369088.0, + "2": 1257388032.0, "3": 1257372160.0, - "4": 1257369600.0, + "4": 1257370112.0, "5": 1257382400.0, - "6": 1257367040.0, - "7": 1257362944.0, - "8": 1257389056.0, - "9": 1257339904.0, - "10": 1257339392.0, - "11": 1257327616.0, - "12": 1257354240.0, - "13": 1257356800.0, - "14": 1257344000.0, - "15": 1257376256.0, - "16": 1257363968.0, - "17": 1257374208.0, + "6": 1257369600.0, + "7": 1257363456.0, + "8": 1257391104.0, + "9": 1257341440.0, + "10": 1257339904.0, + "11": 1257326080.0, + "12": 1257356288.0, + "13": 1257354240.0, + "14": 1257347584.0, + "15": 1257373184.0, + "16": 1257364480.0, + "17": 1257375232.0, "18": 1257352704.0, - "19": 1257348096.0, + "19": 1257348608.0, "20": 1257361408.0, "21": 1257383424.0, - "22": 1257317376.0, - "23": 1257361408.0, - "24": 1257375232.0, - "25": 1257339904.0, - "26": 1257375232.0, - "27": 1257379328.0, - "28": 1257344512.0, - "29": 1257361920.0, + "22": 1257318400.0, + "23": 1257361920.0, + "24": 1257377280.0, + "25": 1257337344.0, + "26": 1257374208.0, + "27": 1257379840.0, + "28": 1257345024.0, + "29": 1257360896.0, "30": 1257371136.0, - "31": 1257334272.0, - "32": 1257356288.0, - "33": 1257341440.0, + "31": 1257333760.0, + "32": 1257354752.0, + "33": 1257340928.0, "34": 1257382400.0, - "35": 1257312256.0, - "36": 1257382912.0, + "35": 1257311232.0, + "36": 1257384960.0, "37": 1257352704.0, - "38": 1257382400.0, - "39": 1257344000.0, + "38": 1257378304.0, + "39": 1257342464.0, "40": 1257394176.0, "41": 1257356800.0, - "42": 1257335808.0, - "43": 1257340416.0, - "44": 1257348608.0, - "45": 1257361920.0, - "46": 1257313792.0, + "42": 1257336832.0, + "43": 1257338368.0, + "44": 1257350656.0, + "45": 1257364480.0, + "46": 1257312256.0, "47": 1257336832.0, - "48": 1257327616.0, - "49": 1257357824.0, - "50": 1257340928.0, - "51": 1257310208.0, - "52": 1257357824.0, - "53": 1257312768.0, - "54": 1257373696.0, - "55": 1257321472.0, + "48": 1257326592.0, + "49": 1257357312.0, + "50": 1257341440.0, + "51": 1257309696.0, + "52": 1257357312.0, + "53": 1257314304.0, + "54": 1257373184.0, + "55": 1257322496.0, "56": 1257371136.0, - "57": 1257341952.0, - "58": 1257348096.0, - "59": 1257347584.0, - "60": 1257311744.0, + "57": 1257340928.0, + "58": 1257348608.0, + "59": 1257349120.0, + "60": 1257313792.0, "61": 1257295360.0, - "62": 1257329152.0, - "63": 1257332224.0, + "62": 1257327104.0, + "63": 1257333248.0, "64": 1257348608.0, - "65": 1257352192.0, - "66": 1257273856.0, + "65": 1257353216.0, + "66": 1257275392.0, "67": 1257360384.0, - "68": 1257308160.0, - "69": 1257333248.0, - "70": 1257301504.0, - "71": 1257320960.0, - "72": 1257318400.0, - "73": 1257319936.0, - "74": 1257283584.0, - "75": 1257308160.0, - "76": 1257313280.0, + "68": 1257307136.0, + "69": 1257333760.0, + "70": 1257299968.0, + "71": 1257321984.0, + "72": 1257320448.0, + "73": 1257316864.0, + "74": 1257284608.0, + "75": 1257309696.0, + "76": 1257312256.0, "77": 1257349632.0, - "78": 1257342976.0, - "79": 1257314304.0, - "80": 1257260032.0, - "81": 1257303552.0, - "82": 1257275392.0, + "78": 1257344512.0, + "79": 1257313280.0, + "80": 1257257984.0, + "81": 1257302528.0, + "82": 1257277952.0, "83": 1257269248.0, - "84": 1257310720.0, + "84": 1257312768.0, "85": 1257284096.0, - "86": 1257277952.0, - "87": 1257269248.0, - "88": 1257281024.0, - "89": 1257268224.0, - "90": 1257305600.0, - "91": 1257270784.0, - "92": 1257271808.0, - "93": 1257287680.0, - "94": 1257270272.0, - "95": 1257265152.0, - "96": 1257272320.0, - "97": 1257265152.0, + "86": 1257276928.0, + "87": 1257267712.0, + "88": 1257282560.0, + "89": 1257268736.0, + "90": 1257306112.0, + "91": 1257267712.0, + "92": 1257269760.0, + "93": 1257285120.0, + "94": 1257268736.0, + "95": 1257267712.0, + "96": 1257271808.0, + "97": 1257264640.0, "98": 1257313792.0, "99": 1257251328.0, - "100": 1257255936.0 + "100": 1257255424.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3479111680.0, - "2": 3875777024.0, - "3": 3875777024.0, - "4": 3875777024.0, - "5": 3875777024.0, - "6": 3875777024.0, - "7": 3875777024.0, - "8": 3875777024.0, - "9": 3875777024.0, - "10": 3875777024.0, - "11": 3875777024.0, - "12": 3875777024.0, - "13": 3875777024.0, - "14": 3875777024.0, - "15": 3875777024.0, - "16": 3875777024.0, - "17": 3875777024.0, - "18": 3875777024.0, - "19": 3875777024.0, - "20": 3875777024.0, - "21": 3875777024.0, - "22": 3875777024.0, - "23": 3875777024.0, - "24": 3875777024.0, - "25": 3875777024.0, - "26": 3875777024.0, - "27": 3875777024.0, - "28": 3875777024.0, - "29": 3875777024.0, - "30": 3875777024.0, - "31": 3875777024.0, - "32": 3875777024.0, - "33": 3875777024.0, - "34": 3875777024.0, - "35": 3875777024.0, - "36": 3875777024.0, - "37": 3875777024.0, - "38": 3875777024.0, - "39": 3875777024.0, - "40": 3875777024.0, - "41": 3875777024.0, - "42": 3875777024.0, - "43": 3875777024.0, - "44": 3875777024.0, - "45": 3875777024.0, - "46": 3875777024.0, - "47": 3875777024.0, - "48": 3875777024.0, - "49": 3875777024.0, - "50": 3875777024.0, - "51": 3875777024.0, - "52": 3875777024.0, - "53": 3875777024.0, - "54": 3875777024.0, - "55": 3875777024.0, - "56": 3875777024.0, - "57": 3875777024.0, - "58": 3875777024.0, - "59": 3875777024.0, - "60": 3875777024.0, - "61": 3875777024.0, - "62": 3875777024.0, - "63": 3875777024.0, - "64": 3875777024.0, - "65": 3875777024.0, - "66": 3875777024.0, - "67": 3875777024.0, - "68": 3875777024.0, - "69": 3875777024.0, - "70": 3875777024.0, - "71": 3875777024.0, - "72": 3875777024.0, - "73": 3875777024.0, - "74": 3875777024.0, - "75": 3875777024.0, - "76": 3875777024.0, - "77": 3875777024.0, - "78": 3875777024.0, - "79": 3875777024.0, - "80": 3875777024.0, - "81": 3875777024.0, - "82": 3875777024.0, - "83": 3875777024.0, - "84": 3875777024.0, - "85": 3875777024.0, - "86": 3875777024.0, - "87": 3875777024.0, - "88": 3875777024.0, - "89": 3875777024.0, - "90": 3875777024.0, - "91": 3875777024.0, - "92": 3875777024.0, - "93": 3875777024.0, - "94": 3875777024.0, - "95": 3875777024.0, - "96": 3875777024.0, - "97": 3875777024.0, - "98": 3875777024.0, - "99": 3875777024.0, - "100": 3875777024.0 + "1": 3479515648.0, + "2": 3875144192.0, + "3": 3875144192.0, + "4": 3875144192.0, + "5": 3875144192.0, + "6": 3875144192.0, + "7": 3875144192.0, + "8": 3875144192.0, + "9": 3875144192.0, + "10": 3875144192.0, + "11": 3875144192.0, + "12": 3875144192.0, + "13": 3875144192.0, + "14": 3875144192.0, + "15": 3875144192.0, + "16": 3875144192.0, + "17": 3875144192.0, + "18": 3875144192.0, + "19": 3875144192.0, + "20": 3875144192.0, + "21": 3875144192.0, + "22": 3875144192.0, + "23": 3875144192.0, + "24": 3875144192.0, + "25": 3875144192.0, + "26": 3875144192.0, + "27": 3875144192.0, + "28": 3875144192.0, + "29": 3875144192.0, + "30": 3875144192.0, + "31": 3875144192.0, + "32": 3875144192.0, + "33": 3875144192.0, + "34": 3875144192.0, + "35": 3875144192.0, + "36": 3875144192.0, + "37": 3875144192.0, + "38": 3875144192.0, + "39": 3875144192.0, + "40": 3875144192.0, + "41": 3875144192.0, + "42": 3875144192.0, + "43": 3875144192.0, + "44": 3875144192.0, + "45": 3875144192.0, + "46": 3875144192.0, + "47": 3875144192.0, + "48": 3875144192.0, + "49": 3875144192.0, + "50": 3875144192.0, + "51": 3875144192.0, + "52": 3875144192.0, + "53": 3875144192.0, + "54": 3875144192.0, + "55": 3875144192.0, + "56": 3875144192.0, + "57": 3875144192.0, + "58": 3875144192.0, + "59": 3875144192.0, + "60": 3875144192.0, + "61": 3875144192.0, + "62": 3875144192.0, + "63": 3875144192.0, + "64": 3875144192.0, + "65": 3875144192.0, + "66": 3875144192.0, + "67": 3875144192.0, + "68": 3875144192.0, + "69": 3875144192.0, + "70": 3875144192.0, + "71": 3875144192.0, + "72": 3875144192.0, + "73": 3875144192.0, + "74": 3875144192.0, + "75": 3875144192.0, + "76": 3875144192.0, + "77": 3875144192.0, + "78": 3875144192.0, + "79": 3875144192.0, + "80": 3875144192.0, + "81": 3875144192.0, + "82": 3875144192.0, + "83": 3875144192.0, + "84": 3875144192.0, + "85": 3875144192.0, + "86": 3875144192.0, + "87": 3875144192.0, + "88": 3875144192.0, + "89": 3875144192.0, + "90": 3875144192.0, + "91": 3875144192.0, + "92": 3875144192.0, + "93": 3875144192.0, + "94": 3875144192.0, + "95": 3875144192.0, + "96": 3875144192.0, + "97": 3875144192.0, + "98": 3875144192.0, + "99": 3875144192.0, + "100": 3875144192.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.74055, - "3": 0.71481, - "4": 0.71002, - "5": 0.70023, - "6": 0.80353, - "7": 1.08069, - "8": 0.82966, - "9": 0.68616, - "10": 0.69352, - "11": 1.3763, - "12": 0.67456, - "13": 0.68103, - "14": 0.68088, - "15": 0.68111, - "16": 0.6849, - "17": 0.6821, - "18": 0.67628, - "19": 0.67892, - "20": 0.67114, - "21": 0.6794, - "22": 0.67588, - "23": 0.66658, - "24": 0.68384, - "25": 0.67495, - "26": 0.67535, - "27": 0.67941, - "28": 0.67459, - "29": 0.67946, - "30": 0.6735, - "31": 0.67923, - "32": 0.67654, - "33": 0.66445, - "34": 0.67321, - "35": 0.67477, - "36": 0.66866, - "37": 0.66017, - "38": 0.67265, - "39": 0.67363, - "40": 0.67488, - "41": 0.67745, - "42": 0.67349, - "43": 0.6743, - "44": 0.67482, - "45": 0.68638, - "46": 0.67075, - "47": 0.67307, - "48": 0.67068, - "49": 0.6716, - "50": 0.67375, - "51": 0.75782, - "52": 0.70259, - "53": 0.68514, - "54": 0.66848, - "55": 0.68164, - "56": 0.66032, - "57": 0.66699, - "58": 0.67902, - "59": 0.66589, - "60": 0.66725, - "61": 0.66884, - "62": 0.69066, - "63": 0.66959, - "64": 0.67724, - "65": 0.68353, - "66": 0.68327, - "67": 0.6822, - "68": 0.6836, - "69": 0.68627, - "70": 0.74292, - "71": 0.66905, - "72": 0.67629, - "73": 0.67447, - "74": 0.66723, - "75": 0.68207, - "76": 0.66991, - "77": 0.65869, - "78": 1.02625, - "79": 0.66065, - "80": 0.67113, - "81": 0.66751, - "82": 0.78247, - "83": 0.73645, - "84": 0.67068, - "85": 0.66462, - "86": 0.66825, - "87": 0.671, - "88": 0.66284, - "89": 0.66802, - "90": 0.66231, - "91": 0.66494, - "92": 0.67077, - "93": 0.66016, - "94": 0.66796, - "95": 0.66874, - "96": 0.66957, - "97": 0.66025, - "98": 0.65909, - "99": 0.70097, - "100": 0.68106 + "2": 5.05197, + "3": 0.63827, + "4": 0.61428, + "5": 0.60177, + "6": 0.59625, + "7": 0.83696, + "8": 0.68113, + "9": 0.70438, + "10": 0.73208, + "11": 1.28744, + "12": 0.5842, + "13": 0.58798, + "14": 0.82864, + "15": 0.57902, + "16": 0.84804, + "17": 0.58862, + "18": 0.98875, + "19": 0.58674, + "20": 0.58856, + "21": 0.81221, + "22": 0.64882, + "23": 0.71221, + "24": 0.59437, + "25": 0.59414, + "26": 0.59745, + "27": 0.59912, + "28": 0.64814, + "29": 0.5895, + "30": 0.58196, + "31": 0.59531, + "32": 0.94185, + "33": 0.61457, + "34": 0.80724, + "35": 0.58887, + "36": 0.5882, + "37": 0.59323, + "38": 0.59338, + "39": 0.59129, + "40": 0.59482, + "41": 0.59316, + "42": 0.58883, + "43": 0.58281, + "44": 0.7635, + "45": 0.59319, + "46": 0.59241, + "47": 0.59301, + "48": 0.78299, + "49": 0.8289, + "50": 0.58541, + "51": 0.64392, + "52": 0.63727, + "53": 0.59183, + "54": 0.58993, + "55": 0.59272, + "56": 0.58993, + "57": 0.58902, + "58": 0.58645, + "59": 0.58766, + "60": 0.59652, + "61": 0.59505, + "62": 0.58854, + "63": 0.59083, + "64": 0.8264, + "65": 0.59174, + "66": 0.59209, + "67": 0.58332, + "68": 0.59283, + "69": 0.59916, + "70": 0.5963, + "71": 0.59264, + "72": 0.58712, + "73": 0.60364, + "74": 0.58757, + "75": 0.59904, + "76": 0.59427, + "77": 0.59296, + "78": 0.8941, + "79": 0.59702, + "80": 0.59468, + "81": 0.59448, + "82": 0.68563, + "83": 0.65036, + "84": 0.64256, + "85": 0.58971, + "86": 0.58161, + "87": 0.60081, + "88": 0.72432, + "89": 0.6952, + "90": 0.58891, + "91": 0.58933, + "92": 0.5939, + "93": 0.58934, + "94": 0.58253, + "95": 0.59991, + "96": 0.58981, + "97": 0.58797, + "98": 0.59814, + "99": 1.22506, + "100": 0.59972 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 2dc643a10f8..b99d883df98 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92183, - "2": 10.9077, - "3": 10.90928, - "4": 10.9197, - "5": 10.90609, - "6": 10.91532, - "7": 10.92252, - "8": 10.90995, - "9": 10.91161, - "10": 10.90689, - "11": 10.88851, - "12": 10.90503, - "13": 10.89514, - "14": 10.89598, - "15": 10.87477, - "16": 10.86137, - "17": 10.86454, - "18": 10.85029, - "19": 10.85483, - "20": 10.78398, - "21": 10.77763, - "22": 10.75638, - "23": 10.74498, - "24": 10.71724, - "25": 10.71549, - "26": 10.71018, - "27": 10.67229, - "28": 10.60467, - "29": 10.57062, - "30": 10.54185, - "31": 10.54427, - "32": 10.53161, - "33": 10.49774, - "34": 10.47495, + "1": 10.92169, + "2": 10.90762, + "3": 10.90909, + "4": 10.91955, + "5": 10.90573, + "6": 10.91571, + "7": 10.92287, + "8": 10.90904, + "9": 10.91203, + "10": 10.90702, + "11": 10.88873, + "12": 10.90475, + "13": 10.89504, + "14": 10.89573, + "15": 10.87518, + "16": 10.8614, + "17": 10.86441, + "18": 10.85053, + "19": 10.85499, + "20": 10.78455, + "21": 10.7775, + "22": 10.75611, + "23": 10.74483, + "24": 10.71735, + "25": 10.71547, + "26": 10.71001, + "27": 10.67234, + "28": 10.60468, + "29": 10.57052, + "30": 10.54195, + "31": 10.54485, + "32": 10.53174, + "33": 10.49752, + "34": 10.47464, "35": 10.47083, - "36": 10.44074, - "37": 10.41615, - "38": 10.41757, - "39": 10.38481, - "40": 10.36819, - "41": 10.34676, - "42": 10.33402, - "43": 10.31355, - "44": 10.28912, - "45": 10.29322, - "46": 10.26555, - "47": 10.24535, - "48": 10.19867, - "49": 10.19508, + "36": 10.44096, + "37": 10.41642, + "38": 10.41727, + "39": 10.38507, + "40": 10.36831, + "41": 10.34679, + "42": 10.33436, + "43": 10.31332, + "44": 10.28897, + "45": 10.29332, + "46": 10.26565, + "47": 10.24514, + "48": 10.19831, + "49": 10.19477, "50": 10.1961, - "51": 10.20234, - "52": 10.15458, - "53": 10.16274, - "54": 10.12749, - "55": 10.09557, - "56": 10.1248, - "57": 10.11363, - "58": 10.12436, - "59": 10.07391, - "60": 10.0937, - "61": 10.04301, - "62": 10.0162, - "63": 10.08729, - "64": 10.03767, + "51": 10.20232, + "52": 10.15459, + "53": 10.16262, + "54": 10.12729, + "55": 10.09552, + "56": 10.12465, + "57": 10.11344, + "58": 10.12412, + "59": 10.07409, + "60": 10.09369, + "61": 10.04308, + "62": 10.0163, + "63": 10.08727, + "64": 10.03778, "65": 10.01755, - "66": 10.04075, + "66": 10.04058, "67": 10.01819, - "68": 9.97999, - "69": 10.00344, - "70": 9.98385, - "71": 10.01034, - "72": 9.99152, - "73": 9.98086, - "74": 9.96234, - "75": 9.94256, - "76": 9.97138, - "77": 9.96086, - "78": 9.91565, - "79": 9.92236, - "80": 9.93093, - "81": 9.9588, - "82": 9.89549, - "83": 9.8631, - "84": 9.79593, - "85": 9.78814, + "68": 9.9801, + "69": 10.00352, + "70": 9.98393, + "71": 10.01032, + "72": 9.99173, + "73": 9.98076, + "74": 9.9623, + "75": 9.94249, + "76": 9.97148, + "77": 9.96097, + "78": 9.91558, + "79": 9.92241, + "80": 9.93107, + "81": 9.95879, + "82": 9.89539, + "83": 9.86306, + "84": 9.79603, + "85": 9.78813, "86": 9.88181, "87": 9.90776, "88": 9.88258, - "89": 9.82994, - "90": 9.82197, - "91": 9.82939, - "92": 9.82025, - "93": 9.75994, - "94": 9.83142, - "95": 9.82809, - "96": 9.80747, - "97": 9.75381, - "98": 9.78418, - "99": 9.82603, - "100": 9.71824 + "89": 9.82999, + "90": 9.82224, + "91": 9.82944, + "92": 9.82018, + "93": 9.76029, + "94": 9.83143, + "95": 9.82804, + "96": 9.80736, + "97": 9.75384, + "98": 9.78406, + "99": 9.82607, + "100": 9.7184 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2443.0, - "2": 2426.0, - "3": 2525.0, - "4": 2507.0, - "5": 2396.0, - "6": 2390.0, - "7": 2713.0, - "8": 2450.0, - "9": 2503.0, - "10": 2522.0, - "11": 2501.0, - "12": 2415.0, - "13": 2653.0, - "14": 2579.0, - "15": 2431.0, - "16": 2402.0, - "17": 2493.0, - "18": 2552.0, - "19": 2604.0, - "20": 2483.0, - "21": 2598.0, - "22": 2493.0, - "23": 2503.0, - "24": 2584.0, - "25": 2435.0, - "26": 2468.0, - "27": 2494.0, - "28": 2636.0, - "29": 2684.0, - "30": 2519.0, - "31": 2839.0, - "32": 2808.0, - "33": 6456.0, - "34": 2874.0, - "35": 527564.0, - "36": 527663.0, - "37": 26089.0, - "38": 527761.0, - "39": 527729.0, - "40": 527902.0, - "41": 31694.0, - "42": 527567.0, - "43": 528026.0, - "44": 527859.0, - "45": 30402.0, - "46": 528061.0, - "47": 528003.0, - "48": 528169.0, - "49": 528568.0, - "50": 550768.0, - "51": 531413.0, - "52": 533217.0, - "53": 1059324.0, - "54": 554963.0, - "55": 554219.0, - "56": 558038.0, - "57": 1052795.0, - "58": 1078748.0, - "59": 558355.0, - "60": 582120.0, - "61": 587284.0, - "62": 1578182.0, - "63": 557832.0, - "64": 557805.0, - "65": 1578376.0, - "66": 1055033.0, - "67": 1085742.0, - "68": 1053875.0, - "69": 561563.0, - "70": 531055.0, - "71": 1054821.0, - "72": 550583.0, - "73": 1578398.0, - "74": 1075460.0, - "75": 557948.0, - "76": 1578104.0, - "77": 550589.0, - "78": 556345.0, - "79": 1077900.0, - "80": 578114.0, - "81": 1059016.0, - "82": 1055102.0, - "83": 1577764.0, - "84": 575310.0, - "85": 556803.0, - "86": 554852.0, - "87": 575521.0, - "88": 559536.0, - "89": 1053598.0, - "90": 1578544.0, - "91": 1053540.0, - "92": 1053497.0, - "93": 35286.0, - "94": 1053482.0, - "95": 530157.0, - "96": 1055299.0, - "97": 1054056.0, - "98": 1054267.0, - "99": 577154.0, - "100": 1578785.0 + "1": 2468.0, + "2": 2401.0, + "3": 2493.0, + "4": 2505.0, + "5": 2492.0, + "6": 2441.0, + "7": 2647.0, + "8": 2461.0, + "9": 2625.0, + "10": 2517.0, + "11": 2523.0, + "12": 2365.0, + "13": 2673.0, + "14": 2542.0, + "15": 2403.0, + "16": 2464.0, + "17": 2630.0, + "18": 2450.0, + "19": 2642.0, + "20": 2426.0, + "21": 2654.0, + "22": 2624.0, + "23": 2472.0, + "24": 2570.0, + "25": 2385.0, + "26": 2633.0, + "27": 2532.0, + "28": 2541.0, + "29": 2652.0, + "30": 2563.0, + "31": 2722.0, + "32": 2789.0, + "33": 3973.0, + "34": 2782.0, + "35": 527627.0, + "36": 527593.0, + "37": 25917.0, + "38": 527689.0, + "39": 527839.0, + "40": 527891.0, + "41": 31727.0, + "42": 527517.0, + "43": 528077.0, + "44": 527852.0, + "45": 30413.0, + "46": 527985.0, + "47": 528131.0, + "48": 528209.0, + "49": 528359.0, + "50": 550860.0, + "51": 560328.0, + "52": 533340.0, + "53": 1055795.0, + "54": 555563.0, + "55": 554263.0, + "56": 1053101.0, + "57": 1052681.0, + "58": 1079130.0, + "59": 557347.0, + "60": 582286.0, + "61": 587119.0, + "62": 1578262.0, + "63": 557738.0, + "64": 537139.0, + "65": 1578266.0, + "66": 1058838.0, + "67": 1056227.0, + "68": 1053806.0, + "69": 561478.0, + "70": 531123.0, + "71": 1053985.0, + "72": 1053905.0, + "73": 1075978.0, + "74": 1075340.0, + "75": 557909.0, + "76": 1578167.0, + "77": 552293.0, + "78": 556281.0, + "79": 1077972.0, + "80": 578039.0, + "81": 1058970.0, + "82": 1080698.0, + "83": 1084664.0, + "84": 575523.0, + "85": 560438.0, + "86": 554843.0, + "87": 575549.0, + "88": 536306.0, + "89": 1053569.0, + "90": 1578469.0, + "91": 1053432.0, + "92": 1053657.0, + "93": 529105.0, + "94": 1053708.0, + "95": 558441.0, + "96": 1055928.0, + "97": 1054101.0, + "98": 1054213.0, + "99": 554460.0, + "100": 1578715.0 } }, "mem-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1322251776.0, - "2": 1614032384.0, - "3": 1617393664.0, - "4": 1618344960.0, - "5": 1618880512.0, - "6": 1618880512.0, - "7": 1618880512.0, - "8": 1618880512.0, - "9": 1618880512.0, - "10": 1618880512.0, - "11": 1618880512.0, - "12": 1618880512.0, - "13": 1618880512.0, - "14": 1618880512.0, - "15": 1618880512.0, - "16": 1618880512.0, - "17": 1618880512.0, - "18": 1618880512.0, - "19": 1618880512.0, - "20": 1618880512.0, - "21": 1618880512.0, - "22": 1618880512.0, - "23": 1618880512.0, - "24": 1618880512.0, - "25": 1618880512.0, - "26": 1618880512.0, - "27": 1618880512.0, - "28": 1618880512.0, - "29": 1618880512.0, - "30": 1618880512.0, - "31": 1618880512.0, - "32": 1618880512.0, - "33": 1618880512.0, - "34": 1618880512.0, - "35": 1618880512.0, - "36": 1618880512.0, - "37": 1618880512.0, - "38": 1618880512.0, - "39": 1618880512.0, - "40": 1618880512.0, - "41": 1618880512.0, - "42": 1618880512.0, - "43": 1618880512.0, - "44": 1618880512.0, - "45": 1618880512.0, - "46": 1618880512.0, - "47": 1618880512.0, - "48": 1618880512.0, - "49": 1618880512.0, - "50": 1618880512.0, - "51": 1618880512.0, - "52": 1618880512.0, - "53": 1618880512.0, - "54": 1618880512.0, - "55": 1618880512.0, - "56": 1618880512.0, - "57": 1618880512.0, - "58": 1618880512.0, - "59": 1618880512.0, - "60": 1618880512.0, - "61": 1618880512.0, - "62": 1618880512.0, - "63": 1618880512.0, - "64": 1618880512.0, - "65": 1618880512.0, - "66": 1618880512.0, - "67": 1618880512.0, - "68": 1618880512.0, - "69": 1618880512.0, - "70": 1618880512.0, - "71": 1618880512.0, - "72": 1618880512.0, - "73": 1618880512.0, - "74": 1618880512.0, - "75": 1618880512.0, - "76": 1618880512.0, - "77": 1618880512.0, - "78": 1618880512.0, - "79": 1618880512.0, - "80": 1618880512.0, - "81": 1618880512.0, - "82": 1618880512.0, - "83": 1618880512.0, - "84": 1618880512.0, - "85": 1618880512.0, - "86": 1618880512.0, - "87": 1618880512.0, - "88": 1618880512.0, - "89": 1618880512.0, - "90": 1618880512.0, - "91": 1618880512.0, - "92": 1618880512.0, - "93": 1618880512.0, - "94": 1618880512.0, - "95": 1618880512.0, - "96": 1618880512.0, - "97": 1618880512.0, - "98": 1618880512.0, - "99": 1618880512.0, - "100": 1618880512.0 + "1": 1323515904.0, + "2": 1613952512.0, + "3": 1617445376.0, + "4": 1619704832.0, + "5": 1619704832.0, + "6": 1619704832.0, + "7": 1619704832.0, + "8": 1619704832.0, + "9": 1619704832.0, + "10": 1619704832.0, + "11": 1619704832.0, + "12": 1619704832.0, + "13": 1619704832.0, + "14": 1619704832.0, + "15": 1619704832.0, + "16": 1619704832.0, + "17": 1619704832.0, + "18": 1619704832.0, + "19": 1619704832.0, + "20": 1619704832.0, + "21": 1619704832.0, + "22": 1619704832.0, + "23": 1619704832.0, + "24": 1619704832.0, + "25": 1619704832.0, + "26": 1619704832.0, + "27": 1619704832.0, + "28": 1619704832.0, + "29": 1619704832.0, + "30": 1619704832.0, + "31": 1619704832.0, + "32": 1619704832.0, + "33": 1619704832.0, + "34": 1619704832.0, + "35": 1619704832.0, + "36": 1619704832.0, + "37": 1619704832.0, + "38": 1619704832.0, + "39": 1619704832.0, + "40": 1619704832.0, + "41": 1619704832.0, + "42": 1619704832.0, + "43": 1619704832.0, + "44": 1619704832.0, + "45": 1619704832.0, + "46": 1619704832.0, + "47": 1619704832.0, + "48": 1619704832.0, + "49": 1619704832.0, + "50": 1619704832.0, + "51": 1619704832.0, + "52": 1619704832.0, + "53": 1619704832.0, + "54": 1619704832.0, + "55": 1619704832.0, + "56": 1619704832.0, + "57": 1619704832.0, + "58": 1619704832.0, + "59": 1619704832.0, + "60": 1619704832.0, + "61": 1619704832.0, + "62": 1619704832.0, + "63": 1619704832.0, + "64": 1619704832.0, + "65": 1619704832.0, + "66": 1619704832.0, + "67": 1619704832.0, + "68": 1619704832.0, + "69": 1619704832.0, + "70": 1619704832.0, + "71": 1619704832.0, + "72": 1619704832.0, + "73": 1619704832.0, + "74": 1619704832.0, + "75": 1619704832.0, + "76": 1619704832.0, + "77": 1619704832.0, + "78": 1619704832.0, + "79": 1619704832.0, + "80": 1619704832.0, + "81": 1619704832.0, + "82": 1619704832.0, + "83": 1619704832.0, + "84": 1619704832.0, + "85": 1619704832.0, + "86": 1619704832.0, + "87": 1619704832.0, + "88": 1619704832.0, + "89": 1619704832.0, + "90": 1619704832.0, + "91": 1619704832.0, + "92": 1619704832.0, + "93": 1619704832.0, + "94": 1619704832.0, + "95": 1619704832.0, + "96": 1619704832.0, + "97": 1619704832.0, + "98": 1619704832.0, + "99": 1619704832.0, + "100": 1619704832.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.84271, - "3": 2.61896, - "4": 2.13682, - "5": 1.83117, - "6": 2.30687, - "7": 1.83781, - "8": 2.52546, - "9": 2.51587, - "10": 2.47099, - "11": 2.29751, - "12": 2.29851, - "13": 2.16383, - "14": 2.60155, - "15": 2.35852, - "16": 2.23005, - "17": 2.70719, - "18": 2.26641, - "19": 2.34687, - "20": 2.20341, - "21": 2.61366, - "22": 1.92204, - "23": 2.41045, - "24": 2.20076, - "25": 2.57863, - "26": 2.61356, - "27": 1.90975, - "28": 2.39018, - "29": 2.17417, - "30": 2.02774, - "31": 1.66802, - "32": 3.19679, - "33": 1.94532, - "34": 1.78092, - "35": 2.86854, - "36": 2.52329, - "37": 2.25087, - "38": 1.89933, - "39": 2.74747, - "40": 2.2918, - "41": 2.7229, - "42": 3.80161, - "43": 2.04029, - "44": 2.71436, - "45": 2.62618, - "46": 2.50912, - "47": 2.40089, - "48": 2.62069, - "49": 2.38881, - "50": 3.11118, - "51": 1.83181, - "52": 2.38462, - "53": 2.18831, - "54": 2.47774, - "55": 2.16497, - "56": 2.71397, - "57": 1.7184, - "58": 2.63468, - "59": 2.27191, - "60": 2.01431, - "61": 2.90864, - "62": 2.68672, - "63": 1.73088, - "64": 2.4613, - "65": 2.16857, - "66": 2.47629, - "67": 2.56543, - "68": 1.74081, - "69": 2.53246, - "70": 2.28935, - "71": 2.87404, - "72": 2.03657, - "73": 2.33536, - "74": 2.08138, - "75": 2.2141, - "76": 2.51275, - "77": 2.10365, - "78": 2.23134, - "79": 2.75405, - "80": 1.97207, - "81": 2.79415, - "82": 2.49662, - "83": 2.37386, - "84": 3.33686, - "85": 2.35332, - "86": 2.33909, - "87": 3.14205, - "88": 2.91235, - "89": 2.71384, - "90": 2.6279, - "91": 2.49967, - "92": 2.99414, - "93": 1.89073, - "94": 3.46814, - "95": 2.23133, - "96": 2.45399, - "97": 1.89708, - "98": 2.16621, - "99": 3.06519, - "100": 2.17963 + "2": 7.68514, + "3": 2.41109, + "4": 2.12571, + "5": 1.7106, + "6": 2.3396, + "7": 1.56367, + "8": 2.49187, + "9": 2.40869, + "10": 2.22199, + "11": 2.15905, + "12": 2.18905, + "13": 2.06518, + "14": 2.28406, + "15": 2.0188, + "16": 1.9362, + "17": 2.82116, + "18": 2.00201, + "19": 2.51471, + "20": 2.27688, + "21": 2.6674, + "22": 1.93988, + "23": 2.22669, + "24": 2.04917, + "25": 2.34928, + "26": 2.52125, + "27": 1.82711, + "28": 2.08376, + "29": 1.97763, + "30": 2.02156, + "31": 1.57399, + "32": 3.13154, + "33": 1.74663, + "34": 1.74915, + "35": 2.54337, + "36": 2.05802, + "37": 2.04946, + "38": 1.86956, + "39": 2.70488, + "40": 2.20313, + "41": 2.89261, + "42": 2.61467, + "43": 1.91714, + "44": 2.64018, + "45": 2.65462, + "46": 2.05718, + "47": 2.27296, + "48": 2.43737, + "49": 2.10218, + "50": 2.3108, + "51": 2.04218, + "52": 2.68079, + "53": 2.43444, + "54": 2.54551, + "55": 2.37149, + "56": 3.6745, + "57": 1.46773, + "58": 2.75598, + "59": 2.16655, + "60": 2.06103, + "61": 2.68918, + "62": 3.0389, + "63": 2.1143, + "64": 2.46238, + "65": 2.24472, + "66": 2.73371, + "67": 2.58536, + "68": 1.77933, + "69": 2.43938, + "70": 2.38083, + "71": 3.05416, + "72": 2.40304, + "73": 2.38168, + "74": 2.10704, + "75": 2.25387, + "76": 2.68557, + "77": 2.31786, + "78": 2.33261, + "79": 2.97302, + "80": 2.373, + "81": 3.06796, + "82": 2.75044, + "83": 2.62014, + "84": 2.13969, + "85": 2.33635, + "86": 2.42282, + "87": 3.20071, + "88": 2.83612, + "89": 2.57963, + "90": 2.63758, + "91": 2.64927, + "92": 3.16655, + "93": 2.22728, + "94": 2.80061, + "95": 2.37427, + "96": 2.68893, + "97": 2.31529, + "98": 2.36377, + "99": 3.42956, + "100": 2.23669 } } } \ No newline at end of file diff --git a/tests/test_utils/recipes/gb200/gpt.yaml b/tests/test_utils/recipes/gb200/gpt.yaml index e10cce0cc3c..0360470c18f 100644 --- a/tests/test_utils/recipes/gb200/gpt.yaml +++ b/tests/test_utils/recipes/gb200/gpt.yaml @@ -124,6 +124,11 @@ products: - environment: [dev] scope: [nightly] platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp1_pp2_dsv4_hybrid_fused] + # products: + # - environment: [dev] + # scope: [mr, mr-github, mr-github-slim] + # platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: # - environment: [dev] diff --git a/tests/test_utils/recipes/h100/gpt.yaml b/tests/test_utils/recipes/h100/gpt.yaml index 9e74d25a87c..be2371f1c4d 100644 --- a/tests/test_utils/recipes/h100/gpt.yaml +++ b/tests/test_utils/recipes/h100/gpt.yaml @@ -133,6 +133,11 @@ products: platforms: [dgx_h100] - environment: [lts] scope: [nightly] + - test_case: [gpt3_mcore_te_tp1_pp2_dsv4_hybrid_mhc_mtp] + products: + - environment: [dev] + scope: [mr, mr-github, mr-github-slim] + platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: # - environment: [dev] @@ -367,11 +372,6 @@ products: - environment: [dev] scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_pp2_dsv4_hybrid_mhc_mtp] - products: - - environment: [dev] - scope: [mr, mr-github, mr-github-slim] - platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] diff --git a/tests/unit_tests/distributed/megatron_fsdp/test_mcore_fully_sharded_data_parallel.py b/tests/unit_tests/distributed/megatron_fsdp/test_mcore_fully_sharded_data_parallel.py index 500045871e7..61f20853558 100644 --- a/tests/unit_tests/distributed/megatron_fsdp/test_mcore_fully_sharded_data_parallel.py +++ b/tests/unit_tests/distributed/megatron_fsdp/test_mcore_fully_sharded_data_parallel.py @@ -1,5 +1,6 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import copy +import gc import random import numpy as np @@ -323,6 +324,10 @@ def train_step(model, optimizer, inputs): msg=f"Parameters for {name1} don't match", ) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.synchronize() + def test_fsdp_expt_device_mesh(self): """Test that expt_device_mesh is None for dense models and not None for MoE models.""" if not is_torch_min_version("2.4.0"): @@ -534,6 +539,10 @@ def train_step(model, optimizer, inputs): msg=f"Parameters for {name1} don't match", ) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.synchronize() + @classmethod def hsdp_one_step_test(cls, num_fsdp_group): if not is_torch_min_version("2.4.0"): @@ -856,6 +865,10 @@ def _training_loop(seed=42, **kwargs): dict(data_parallel_sharding_strategy="optim", fsdp_double_buffer=False), id="optim_double_buffer", ), + pytest.param( + dict(data_parallel_sharding_strategy="no_shard", fsdp_double_buffer=False), + id="no_shard", + ), ], ) def test_compatible_with_nd_parallel(self, ref_cache, nd_topology, spec_configs): @@ -872,9 +885,13 @@ def test_compatible_with_nd_parallel(self, ref_cache, nd_topology, spec_configs) use_distributed_optimizer=True, **distopt_spec_configs ) + fsdp_sharding_strategy = spec_configs["data_parallel_sharding_strategy"] + # no_shard is incompatible with meta device initialization. See fully_shard.py:326. + init_model_with_meta_device = fsdp_sharding_strategy != "no_shard" + outputs = TestMegatronFSDPE2E._training_loop( use_megatron_fsdp=True, - init_model_with_meta_device=True, + init_model_with_meta_device=init_model_with_meta_device, ckpt_format="fsdp_dtensor", gradient_accumulation_fusion=False, **spec_configs, @@ -897,6 +914,10 @@ def test_compatible_with_nd_parallel(self, ref_cache, nd_topology, spec_configs) ), ) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.synchronize() + def compare_losses(loss_a: float, loss_b: float, reference: str = "b"): """ diff --git a/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_param_and_grad_buffer.py b/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_param_and_grad_buffer.py new file mode 100644 index 00000000000..8ad3789b449 --- /dev/null +++ b/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_param_and_grad_buffer.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import math + +import torch + +from megatron.core.distributed.fsdp.src.megatron_fsdp.param_and_grad_buffer import ( + BucketingPolicy, + _get_parameter_groups, +) + + +class _ExpertTestModule(torch.nn.Module): + """ + Mock module whose params are routed under `.experts.` to trigger + is_expert_param=True. The outer `layer` attribute puts a dot before + `experts` in the parameter path (e.g. `layer.experts.linear_fc1`). + """ + + def __init__(self, shapes): + super().__init__() + self.layer = torch.nn.Module() + self.layer.experts = torch.nn.ParameterDict( + {name: torch.nn.Parameter(torch.empty(shape)) for name, shape in shapes.items()} + ) + + +def _get_bucket_signatures(module): + bucket_groups, _, _ = _get_parameter_groups( + module, BucketingPolicy(suggested_bucket_size=None), meta_device_init_fp8_params={} + ) + param_to_name = {param: name for name, param in module.named_parameters()} + return [ + { + "chunk_size_factor": group.chunk_size_factor, + "params": [(param_to_name[param], tuple(param.shape)) for param in group.params], + } + for group in bucket_groups + ] + + +def test_grouped_expert_weights_split_when_chunk_size_factors_differ(): + """Grouped expert weights with mismatched chunk size factors get routed to separate buckets.""" + num_local_experts = 4 + hidden_size = 12 + moe_ffn_hidden_size = 8 + shapes = { + "linear_fc1": (num_local_experts, 2 * moe_ffn_hidden_size, hidden_size), + "linear_fc2": (num_local_experts, hidden_size, moe_ffn_hidden_size), + } + module = _ExpertTestModule(shapes) + + assert _get_bucket_signatures(module) == [ + { + "chunk_size_factor": torch.Size(shapes["linear_fc1"])[1:].numel(), + "params": [("layer.experts.linear_fc1", shapes["linear_fc1"])], + }, + { + "chunk_size_factor": torch.Size(shapes["linear_fc2"])[1:].numel(), + "params": [("layer.experts.linear_fc2", shapes["linear_fc2"])], + }, + ] + + +def test_per_expert_2d_weights_merge_via_lcm(): + """Per-expert 2D weights merge into a single bucket via LCM chunk size factor.""" + hidden_size = 12 + moe_ffn_hidden_size = 8 + shapes = { + "linear_fc1": (2 * moe_ffn_hidden_size, hidden_size), + "linear_fc2": (hidden_size, moe_ffn_hidden_size), + } + module = _ExpertTestModule(shapes) + + assert _get_bucket_signatures(module) == [ + { + "chunk_size_factor": math.lcm( + torch.Size(shapes["linear_fc1"])[1:].numel(), + torch.Size(shapes["linear_fc2"])[1:].numel(), + ), + "params": [ + ("layer.experts.linear_fc1", shapes["linear_fc1"]), + ("layer.experts.linear_fc2", shapes["linear_fc2"]), + ], + } + ] diff --git a/tests/unit_tests/fusions/test_fused_mhc_kernels.py b/tests/unit_tests/fusions/test_fused_mhc_kernels.py index 15468df8264..91bfa8808df 100644 --- a/tests/unit_tests/fusions/test_fused_mhc_kernels.py +++ b/tests/unit_tests/fusions/test_fused_mhc_kernels.py @@ -131,6 +131,7 @@ def test_fwd_bwd_vs_torch_reference(self, s, b, n, iters): class TestFusedSinkhorn: + @pytest.mark.flaky_in_dev @_require_cutile @pytest.mark.parametrize("s,b,n,iters", [(2, 4, 4, 5), (1, 1, 2, 10)]) def test_fwd_bwd_vs_reference(self, s, b, n, iters): @@ -189,6 +190,7 @@ def test_fwd_bwd_vs_torch_reference(self, s, b, n, C): class TestFusedHAggregate: + @pytest.mark.flaky_in_dev @_require_cutile @pytest.mark.parametrize("s,b,n,C", [(2, 4, 4, 1024), (1, 1, 2, 256)]) def test_fwd_bwd_vs_reference(self, s, b, n, C): @@ -269,6 +271,7 @@ def _make_inputs(): class TestFusedHPostBDA: + @pytest.mark.flaky_in_dev @_require_cutile @pytest.mark.parametrize("with_bias", [True, False]) @pytest.mark.parametrize("s,b,n,C", [(2, 4, 4, 1024), (1, 2, 2, 256)]) @@ -356,6 +359,7 @@ def test_fwd_bwd_vs_torch_reference(self, M, N, K): class TestFusedProjRms: + @pytest.mark.flaky_in_dev @_require_cutile @pytest.mark.parametrize("M,N,K", [(256, 20, 4096), (64, 8, 512)]) def test_fwd_bwd_vs_reference(self, M, N, K): @@ -481,6 +485,7 @@ def _run_inline_ref(): class TestEndToEndFused: """Full mHC pipeline using fused cuTile kernels (requires cuTile).""" + @pytest.mark.flaky_in_dev @_require_cutile def test_full_pipeline_fwd_bwd(self): from megatron.core.fusions.fused_mhc_kernels import ( diff --git a/tests/unit_tests/models/test_dsa_gpt_mamba_equivalence.py b/tests/unit_tests/models/test_dsa_gpt_mamba_equivalence.py index 229af268a79..9255e4794d5 100644 --- a/tests/unit_tests/models/test_dsa_gpt_mamba_equivalence.py +++ b/tests/unit_tests/models/test_dsa_gpt_mamba_equivalence.py @@ -35,6 +35,7 @@ get_transformer_block_with_experimental_attention_variant_spec, ) from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.models.hybrid.hybrid_block import HyperConnectionHybridLayer from megatron.core.models.hybrid.hybrid_layer_allocation import validate_segment_layers from megatron.core.models.hybrid.hybrid_layer_specs import hybrid_stack_spec from megatron.core.models.hybrid.hybrid_model import HybridModel @@ -642,3 +643,52 @@ def test_moe_record_and_compare_golden_values(self, tp: int, pp: int) -> None: # Verify HybridModel matches golden values _compare_against_golden_values(mamba_logprobs, gpt_logprobs, abs_tol=1e-3) + + +# --------------------------------------------------------------------------- +# mHC HybridModel smoke tests for DeepSeek proxy patterns +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +class TestDSAHybridMHCProxy: + """Smoke-test mHC on DeepSeek-style HybridModel patterns. + + These do not assert GPT/Hybrid numerical equivalence because the current + HybridModel implementation wraps each split hybrid layer at the boundary, + whereas GPT mHC has separate attention and MLP hyper-connections inside a + TransformerLayer. + """ + + def teardown_method(self, method): + Utils.destroy_model_parallel() + + def _enable_mhc(self, config: MLATransformerConfig) -> MLATransformerConfig: + config.enable_hyper_connections = True + config.num_residual_streams = 4 + config.mhc_sinkhorn_iterations = 5 + config.mhc_init_gating_factor = 0.01 + config.hidden_dropout = 0.0 + return config + + def _assert_mhc_model_forward(self, config: MLATransformerConfig, pattern: str) -> None: + Utils.initialize_model_parallel(1, 1) + model_parallel_cuda_manual_seed(42) + model = _build_mamba_model(self._enable_mhc(config), pattern) + assert all(isinstance(layer, HyperConnectionHybridLayer) for layer in model.decoder.layers) + + torch.manual_seed(99) + tokens = torch.randint(0, _VOCAB_SIZE, (_BATCH_SIZE, _SEQ_LEN), device='cuda') + logprobs = _forward_logprobs_pp1(model, tokens) + assert logprobs.shape == (_BATCH_SIZE, _SEQ_LEN - 1) + assert torch.isfinite(logprobs).all() + + def test_dsa_dense_hybrid_mhc_forward(self) -> None: + """DeepSeek-V3.2-style DSA + MLP split pattern runs with mHC.""" + config = _make_dsa_config(num_layers=_NUM_GPT_LAYERS, tp=1, pp=1) + self._assert_mhc_model_forward(config, _MAMBA_PATTERN) + + def test_dsa_moe_hybrid_mhc_forward(self) -> None: + """DeepSeek-V3-style DSA + MoE split pattern runs with mHC.""" + config = _make_dsa_moe_config(num_layers=_NUM_GPT_LAYERS, tp=1, pp=1) + self._assert_mhc_model_forward(config, _MOE_MAMBA_PATTERN) diff --git a/tests/unit_tests/models/test_hybrid_model.py b/tests/unit_tests/models/test_hybrid_model.py index 98a53da0314..d17d244fa8f 100644 --- a/tests/unit_tests/models/test_hybrid_model.py +++ b/tests/unit_tests/models/test_hybrid_model.py @@ -16,17 +16,61 @@ from megatron.core.inference.inference_request import DynamicInferenceRequest from megatron.core.inference.sampling_params import SamplingParams from megatron.core.models.common.embeddings.yarn_rotary_pos_embedding import YarnRotaryEmbedding +from megatron.core.models.hybrid.hybrid_block import ( + HybridStack, + HybridStackSubmodules, + HyperConnectionHybridLayer, +) from megatron.core.models.hybrid.hybrid_layer_specs import hybrid_stack_spec from megatron.core.models.hybrid.hybrid_model import HybridModel from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from megatron.core.transformer import TransformerConfig from megatron.core.transformer.enums import AttnBackend -from megatron.core.transformer.module import Float16Module +from megatron.core.transformer.module import Float16Module, MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec from megatron.core.utils import divide, is_fa_min_version, is_torch_min_version from tests.unit_tests.test_utilities import Utils +class _DummyHybridLayer(MegatronModule): + """Minimal same-shape layer used to test HybridModel/mHC plumbing.""" + + def __init__(self, config: TransformerConfig, layer_number: int, **_kwargs): + super().__init__(config=config) + self.layer_number = layer_number + self.proj = torch.nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.seen_hidden_shapes = [] + + def forward( + self, + hidden_states, + attention_mask=None, + inference_context=None, + packed_seq_params=None, + **_kwargs, + ): + self.seen_hidden_shapes.append(tuple(hidden_states.shape)) + return hidden_states + 0.125 * self.proj(hidden_states) + + +def _get_dummy_hybrid_stack_spec() -> ModuleSpec: + """Build a HybridStack spec whose layer symbols all resolve to dummy layers.""" + dummy_layer_spec = ModuleSpec(module=_DummyHybridLayer) + return ModuleSpec( + module=HybridStack, + params={"post_layer_norm": False}, + submodules=HybridStackSubmodules( + mamba_layer=dummy_layer_spec, + gdn_layer=dummy_layer_spec, + attention_layer=dummy_layer_spec, + dsa_layer=dummy_layer_spec, + mlp_layer=dummy_layer_spec, + moe_layer=dummy_layer_spec, + ), + ) + + class TestHybridModel: def setup_method(self, method): @@ -57,6 +101,176 @@ def test_constructor(self): num_weights = sum([p.numel() for p in self.model.parameters()]) assert num_weights == 1774872 + def test_constructor_with_hyper_connections(self): + model_config = TransformerConfig( + num_layers=3, + hidden_size=256, + num_attention_heads=4, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + ) + model = HybridModel( + config=model_config, + hybrid_stack_spec=hybrid_stack_spec, + vocab_size=100, + max_sequence_length=4, + hybrid_layer_pattern="M*-", + ) + + assert all(isinstance(layer, HyperConnectionHybridLayer) for layer in model.decoder.layers) + assert model.decoder.hc_head_fn.shape == ( + model_config.num_residual_streams, + model_config.hidden_size * model_config.num_residual_streams, + ) + assert model.decoder.hc_head_base.shape == (model_config.num_residual_streams,) + assert model.decoder.hc_head_scale.shape == (1,) + assert "decoder.hc_head_fn" in model.state_dict() + decoder_sharded_state = model.decoder.sharded_state_dict(prefix="decoder.", metadata={}) + assert "decoder.hc_head_fn" in decoder_sharded_state + assert "decoder.hc_head_base" in decoder_sharded_state + assert "decoder.hc_head_scale" in decoder_sharded_state + num_weights = sum([p.numel() for p in model.parameters()]) + assert num_weights > sum([p.numel() for p in self.model.parameters()]) + + def test_hyper_connection_recompute_skips_boundary_bda_checkpoint(self, monkeypatch): + model_config = TransformerConfig( + num_layers=1, + hidden_size=8, + num_attention_heads=1, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + mhc_sinkhorn_iterations=3, + ) + layer = HyperConnectionHybridLayer( + config=model_config, layer=_DummyHybridLayer(model_config, layer_number=1) + ) + hidden_states = torch.randn( + 4, 2, model_config.hidden_size * model_config.num_residual_streams, requires_grad=True + ) + manager = type("_FakeManager", (), {})() + manager.is_last_layer_in_recompute_block = True + seen_bda_managers = [] + + def fake_hyper_connection_forward(hidden_states, mhc_recompute_manager=None): + assert mhc_recompute_manager is manager + s, b, _ = hidden_states.shape + n = model_config.num_residual_streams + c = model_config.hidden_size + aggregated = hidden_states.view(s, b, n, c).mean(dim=2) + h_res = torch.empty(s, b, n, n, dtype=hidden_states.dtype) + h_post = torch.empty(s, b, n, dtype=hidden_states.dtype) + return aggregated, h_res, h_post + + def fake_fused_h_res_h_post_bda( + h_res, + original_residual, + h_post, + layer_output_with_bias, + dropout_prob, + training, + fused, + manager=None, + ): + seen_bda_managers.append(manager) + return original_residual + + monkeypatch.setattr(layer.hyper_connection, "forward", fake_hyper_connection_forward) + monkeypatch.setattr( + layer.hyper_connection, "fused_h_res_h_post_bda", fake_fused_h_res_h_post_bda + ) + + output, _ = layer(hidden_states, attention_mask=None, mhc_recompute_manager=manager) + assert output is hidden_states + assert seen_bda_managers == [None] + + manager.is_last_layer_in_recompute_block = False + layer(hidden_states, attention_mask=None, mhc_recompute_manager=manager) + assert seen_bda_managers[-1] is manager + + def test_forward_with_hyper_connections(self): + model_config = TransformerConfig( + num_layers=3, + hidden_size=256, + num_attention_heads=4, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + ) + model = HybridModel( + config=model_config, + hybrid_stack_spec=hybrid_stack_spec, + vocab_size=100, + max_sequence_length=4, + hybrid_layer_pattern="M*-", + ) + model.cuda() + + sequence_length = model.max_sequence_length + micro_batch_size = 2 + data = list(range(sequence_length)) + input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda() + position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda() + attention_mask = torch.ones( + (micro_batch_size, 1, sequence_length, sequence_length), dtype=bool + ).cuda() + + logits = model.forward( + input_ids=input_ids, position_ids=position_ids, attention_mask=attention_mask + ) + + assert logits.shape[0] == micro_batch_size + assert logits.shape[1] == sequence_length + assert logits.shape[2] == model.vocab_size + + def test_dummy_hybrid_model_with_hyper_connections_forward_backward(self): + model_config = TransformerConfig( + num_layers=3, + hidden_size=32, + num_attention_heads=4, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + mhc_sinkhorn_iterations=3, + ) + model = HybridModel( + config=model_config, + hybrid_stack_spec=_get_dummy_hybrid_stack_spec(), + vocab_size=64, + max_sequence_length=8, + hybrid_layer_pattern="M*-", + parallel_output=False, + ) + + assert all(isinstance(layer, HyperConnectionHybridLayer) for layer in model.decoder.layers) + assert all( + isinstance(layer.inner_layer, _DummyHybridLayer) for layer in model.decoder.layers + ) + + model.cuda() + sequence_length = model.max_sequence_length + micro_batch_size = 2 + data = torch.arange(sequence_length, dtype=torch.int64, device='cuda') + input_ids = data.repeat((micro_batch_size, 1)) + position_ids = data.repeat((micro_batch_size, 1)) + + logits = model.forward(input_ids=input_ids, position_ids=position_ids, attention_mask=None) + + assert logits.shape == (micro_batch_size, sequence_length, model.vocab_size) + assert torch.isfinite(logits).all() + + logits.float().mean().backward() + + for layer in model.decoder.layers: + assert layer.inner_layer.seen_hidden_shapes == [ + (sequence_length, micro_batch_size, model_config.hidden_size) + ] + assert layer.inner_layer.proj.weight.grad is not None + assert layer.hyper_connection.mapping_proj.weight.grad is not None + assert torch.isfinite(layer.inner_layer.proj.weight.grad).all() + assert torch.isfinite(layer.hyper_connection.mapping_proj.weight.grad).all() + def test_set_input_tensor(self): config: TransformerConfig = self.model.config sequence_length = self.model.max_sequence_length diff --git a/tests/unit_tests/models/test_hybrid_moe_model.py b/tests/unit_tests/models/test_hybrid_moe_model.py index b7e8365804a..a84389a8057 100644 --- a/tests/unit_tests/models/test_hybrid_moe_model.py +++ b/tests/unit_tests/models/test_hybrid_moe_model.py @@ -36,6 +36,7 @@ "actual_vocab_size": 131072, "add_bias_linear": False, "add_qkv_bias": False, + "apply_dsa_kernel_fusion": True, "apply_query_key_layer_scaling": False, "apply_residual_connection_post_layernorm": False, "apply_rope_fusion": False, diff --git a/tests/unit_tests/ssm/test_gated_delta_net.py b/tests/unit_tests/ssm/test_gated_delta_net.py index 3eb02442fe9..9782fbda10c 100644 --- a/tests/unit_tests/ssm/test_gated_delta_net.py +++ b/tests/unit_tests/ssm/test_gated_delta_net.py @@ -16,6 +16,7 @@ get_transformer_block_with_experimental_attention_variant_spec, ) from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.ssm.gated_delta_net import GatedDeltaNet from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed @@ -45,6 +46,49 @@ HAVE_FLA = False +def _make_gdn_config(**overrides): + config_kwargs = { + "hidden_size": 128, + "linear_conv_kernel_dim": 2, + "linear_key_head_dim": 32, + "linear_value_head_dim": 32, + "linear_num_key_heads": 4, + "linear_num_value_heads": 8, + "num_layers": 1, + "normalization": "RMSNorm", + "use_cpu_initialization": True, + "layernorm_zero_centered_gamma": True, + "num_attention_heads": 8, + "activation_func": F.silu, + "bf16": True, + "experimental_attention_variant": "gated_delta_net", + "linear_attention_freq": [1], + "transformer_impl": "transformer_engine", + } + config_kwargs.update(overrides) + return TransformerConfig(**config_kwargs) + + +@pytest.mark.parametrize("pre_gated_delta_rule_impl", ["unfused", "fused_streamed", "fused_mega"]) +def test_pre_gated_delta_rule_impl_accepts_gdn_modes(pre_gated_delta_rule_impl): + config = _make_gdn_config(pre_gated_delta_rule_impl=pre_gated_delta_rule_impl) + assert config.pre_gated_delta_rule_impl == pre_gated_delta_rule_impl + + +def test_pre_gated_delta_rule_impl_rejects_invalid_value(): + with pytest.raises(ValueError, match="pre_gated_delta_rule_impl must be one of"): + _make_gdn_config(pre_gated_delta_rule_impl="fused") + + +def test_pre_gated_delta_rule_impl_requires_gdn_variant(): + with pytest.raises(ValueError, match="experimental_attention_variant='gated_delta_net'"): + _make_gdn_config( + experimental_attention_variant=None, + linear_attention_freq=None, + pre_gated_delta_rule_impl="fused_streamed", + ) + + @pytest.mark.parametrize( ("tp_size", "sp", "cp_size"), [(1, False, 1), (2, False, 1), (2, True, 1), (1, False, 2), (2, False, 2), (2, True, 2)], @@ -307,6 +351,462 @@ def test_gpu_forward_thd_padding_correctness(self): self.gdn(hidden_states_thd, None, packed_seq_params=actual_mismatch_params) +@pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.") +@pytest.mark.internal +class TestFusedPreGatedDeltaRule: + + @pytest.fixture(scope='function', autouse=True) + def setup_method(self): + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=1, + ) + model_parallel_cuda_manual_seed(123) + + tp_group = parallel_state.get_tensor_model_parallel_group() + cp_group = parallel_state.get_context_parallel_group() + self.pg_collection = ProcessGroupCollection(tp=tp_group, cp=cp_group) + + self.unfused_gdn = self._build_gdn(pre_gated_delta_rule_impl="unfused") + self.fused_gdn = self._build_gdn(pre_gated_delta_rule_impl="fused_streamed") + self.fused_gdn.load_state_dict(self.unfused_gdn.state_dict()) + + def teardown_method(self): + Utils.destroy_model_parallel() + + def _build_gdn( + self, + pre_gated_delta_rule_impl: str, + *, + deterministic_mode: bool = True, + conv_kernel_dim: int = 2, + ): + transformer_config = TransformerConfig( + hidden_size=256, + linear_conv_kernel_dim=conv_kernel_dim, + linear_key_head_dim=64, + linear_value_head_dim=64, + linear_num_key_heads=4, + linear_num_value_heads=8, + num_layers=1, + normalization="RMSNorm", + use_cpu_initialization=True, + layernorm_zero_centered_gamma=True, + num_attention_heads=8, + activation_func=F.silu, + bf16=True, + tensor_model_parallel_size=1, + context_parallel_size=1, + experimental_attention_variant="gated_delta_net", + linear_attention_freq=[1], + transformer_impl="transformer_engine", + deterministic_mode=deterministic_mode, + pre_gated_delta_rule_impl=pre_gated_delta_rule_impl, + ) + gdn_submodules = get_experimental_attention_variant_module_spec( + config=transformer_config + ).submodules + gdn = GatedDeltaNet( + transformer_config, + submodules=gdn_submodules, + layer_number=1, + bias=False, + conv_bias=False, + conv_init=1.0, + use_qk_l2norm=True, + A_init_range=(1, 16), + pg_collection=self.pg_collection, + ) + return gdn.cuda().bfloat16() + + def _packed_pre_gated_delta_rule_reference(self, gdn, qkvzba, cu_seqlens): + """Run the dense torch reference independently on each packed sequence.""" + + segment_outputs = [[] for _ in range(6)] + for start, end in zip(cu_seqlens[:-1].tolist(), cu_seqlens[1:].tolist()): + outputs = gdn.pre_gated_delta_rule(qkvzba[start:end], batch=1, seq_len=end - start) + for output_list, output in zip(segment_outputs, outputs): + output_list.append(output) + return tuple(torch.cat(outputs, dim=1) for outputs in segment_outputs) + + def _assert_pre_gated_delta_rule_outputs_close( + self, + fused_outputs, + unfused_outputs, + *, + atol: float, + rtol: float, + output_tolerances=None, + ): + """Compare named pre-GDR outputs with optional per-output tolerances.""" + + output_names = ("query", "key", "value", "gate", "beta", "g") + output_tolerances = output_tolerances or {} + for name, fused, unfused in zip(output_names, fused_outputs, unfused_outputs): + output_atol, output_rtol = output_tolerances.get(name, (atol, rtol)) + torch.testing.assert_close( + fused, + unfused, + atol=output_atol, + rtol=output_rtol, + msg=lambda msg, output_name=name: f"{output_name} mismatch: {msg}", + ) + + def test_fused_and_unfused_forward_match(self): + hidden_states = torch.randn( + (32, 2, self.unfused_gdn.config.hidden_size), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + + with torch.no_grad(): + unfused_output, unfused_bias = self.unfused_gdn(hidden_states, None) + fused_output, fused_bias = self.fused_gdn(hidden_states, None) + + torch.testing.assert_close(fused_output, unfused_output, atol=1e-3, rtol=1e-3) + assert fused_bias == unfused_bias + + @pytest.mark.parametrize("pre_gated_delta_rule_impl", ["fused_streamed", "fused_mega"]) + def test_fused_and_unfused_forward_thd_match(self, pre_gated_delta_rule_impl): + unfused_gdn = self._build_gdn( + pre_gated_delta_rule_impl="unfused", + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn = self._build_gdn( + pre_gated_delta_rule_impl=pre_gated_delta_rule_impl, + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn.load_state_dict(unfused_gdn.state_dict()) + + hidden_states = torch.randn( + (32, 1, unfused_gdn.config.hidden_size), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + cu_seqlens = torch.tensor([0, 1, 4, 11, 32], device=torch.cuda.current_device(), dtype=torch.int32) + packed_seq_params = PackedSeqParams( + qkv_format='thd', + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=21, + max_seqlen_kv=21, + total_tokens=hidden_states.shape[0], + ) + assert packed_seq_params.seq_idx is not None + + with torch.no_grad(): + unfused_output, unfused_bias = unfused_gdn( + hidden_states, None, packed_seq_params=packed_seq_params + ) + fused_output, fused_bias = fused_gdn( + hidden_states, None, packed_seq_params=packed_seq_params + ) + + torch.testing.assert_close(fused_output, unfused_output, atol=2e-3, rtol=2e-3) + assert fused_bias == unfused_bias + + def test_fused_and_unfused_forward_thd_padding_match(self): + unfused_gdn = self._build_gdn( + pre_gated_delta_rule_impl="unfused", + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn = self._build_gdn( + pre_gated_delta_rule_impl="fused_streamed", + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn.load_state_dict(unfused_gdn.state_dict()) + + hidden_states = torch.randn( + (12, 1, unfused_gdn.config.hidden_size), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + cu_seqlens = torch.tensor([0, 1, 4, 9], device=torch.cuda.current_device(), dtype=torch.int32) + cu_seqlens_padded = torch.tensor( + [0, 2, 6, 12], device=torch.cuda.current_device(), dtype=torch.int32 + ) + packed_seq_params = PackedSeqParams( + qkv_format='thd', + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + max_seqlen_q=6, + max_seqlen_kv=6, + total_tokens=hidden_states.shape[0], + ) + assert packed_seq_params.seq_idx is not None + + with torch.no_grad(): + unfused_output, unfused_bias = unfused_gdn( + hidden_states, None, packed_seq_params=packed_seq_params + ) + fused_output, fused_bias = fused_gdn( + hidden_states, None, packed_seq_params=packed_seq_params + ) + + torch.testing.assert_close(fused_output, unfused_output, atol=2e-3, rtol=2e-3) + assert fused_bias == unfused_bias + + def test_fused_and_unfused_pre_gated_delta_rule_match(self): + batch = 2 + seq_len = 32 + hidden_states = torch.randn( + (seq_len, batch, self.unfused_gdn.config.hidden_size), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + + with torch.no_grad(): + qkvzba, _ = self.unfused_gdn.in_proj(hidden_states) + unfused_outputs = self.unfused_gdn.pre_gated_delta_rule(qkvzba, batch, seq_len) + fused_outputs = self.fused_gdn._fused_streamed_pre_gated_delta_rule(qkvzba) + + self._assert_pre_gated_delta_rule_outputs_close( + fused_outputs, + unfused_outputs, + atol=1e-3, + rtol=1e-3, + # g uses Triton exp/log softplus in the fused path and torch softplus + # in the reference path, so its direct intermediate parity needs a + # slightly looser relative tolerance than the layout/conv outputs. + output_tolerances={"g": (1e-3, 3e-3)}, + ) + + def test_fused_and_unfused_packed_pre_gated_delta_rule_forward_match(self): + reference_gdn = self._build_gdn( + pre_gated_delta_rule_impl="unfused", + deterministic_mode=True, + conv_kernel_dim=4, + ) + fused_gdn = self._build_gdn( + pre_gated_delta_rule_impl="fused_streamed", + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn.load_state_dict(reference_gdn.state_dict()) + + batch = 1 + cu_seqlens = torch.tensor([0, 1, 4, 6, 11], device=torch.cuda.current_device(), dtype=torch.int32) + seq_len = cu_seqlens[-1].item() + qkvzba = torch.randn( + (seq_len, batch, reference_gdn.in_proj_dim), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + + with torch.no_grad(): + unfused_outputs = self._packed_pre_gated_delta_rule_reference( + reference_gdn, qkvzba, cu_seqlens + ) + fused_outputs = fused_gdn._fused_streamed_pre_gated_delta_rule( + qkvzba, cu_seqlens_q=cu_seqlens + ) + + self._assert_pre_gated_delta_rule_outputs_close( + fused_outputs, unfused_outputs, atol=2e-3, rtol=2e-3 + ) + + def test_fused_and_unfused_packed_pre_gated_delta_rule_backward_match(self): + reference_gdn = self._build_gdn( + pre_gated_delta_rule_impl="unfused", + deterministic_mode=True, + conv_kernel_dim=4, + ) + fused_gdn = self._build_gdn( + pre_gated_delta_rule_impl="fused_streamed", + deterministic_mode=False, + conv_kernel_dim=4, + ) + fused_gdn.load_state_dict(reference_gdn.state_dict()) + + batch = 1 + cu_seqlens = torch.tensor([0, 1, 4, 6, 11], device=torch.cuda.current_device(), dtype=torch.int32) + seq_len = cu_seqlens[-1].item() + qkvzba = torch.randn( + (seq_len, batch, reference_gdn.in_proj_dim), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + qkvzba_unfused = qkvzba.detach().clone().requires_grad_(True) + qkvzba_fused = qkvzba.detach().clone().requires_grad_(True) + + reference_gdn.zero_grad(set_to_none=True) + fused_gdn.zero_grad(set_to_none=True) + + unfused_outputs = self._packed_pre_gated_delta_rule_reference( + reference_gdn, qkvzba_unfused, cu_seqlens + ) + fused_outputs = fused_gdn._fused_streamed_pre_gated_delta_rule( + qkvzba_fused, cu_seqlens_q=cu_seqlens + ) + grad_outputs = [torch.randn_like(output.float()) for output in unfused_outputs] + + unfused_loss = sum( + (output.float() * grad).sum() for output, grad in zip(unfused_outputs, grad_outputs) + ) + fused_loss = sum( + (output.float() * grad).sum() for output, grad in zip(fused_outputs, grad_outputs) + ) + unfused_loss.backward() + fused_loss.backward() + + torch.testing.assert_close(qkvzba_fused.grad, qkvzba_unfused.grad, atol=3e-2, rtol=3e-2) + torch.testing.assert_close( + fused_gdn.conv1d.weight.grad, + reference_gdn.conv1d.weight.grad, + atol=3e-2, + rtol=3e-2, + ) + torch.testing.assert_close(fused_gdn.A_log.grad, reference_gdn.A_log.grad, atol=3e-2, rtol=3e-2) + torch.testing.assert_close( + fused_gdn.dt_bias.grad, reference_gdn.dt_bias.grad, atol=3e-2, rtol=3e-2 + ) + + def test_fused_packed_conv_forward_boundary_isolation(self): + from megatron.core.fusions.fused_pre_gated_delta_rule import ( + fused_streamed_pre_gated_delta_rule, + ) + + seq_len = 5 + boundary = 3 + num_key_heads = 1 + # Keep qkvzba.stride(0) aligned for causal_conv1d's channel-last + # backward guard; the boundary condition under test is independent + # of the value-head repeat factor. + num_value_heads = 4 + key_head_dim = 32 + value_head_dim = 32 + conv_width = 4 + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + v_offset = 2 * qk_channels + k_offset = qk_channels + total_channels = 2 * qk_channels + 2 * v_channels + 2 * num_value_heads + device = torch.cuda.current_device() + + qkvzba = torch.zeros((seq_len, 1, total_channels), device=device, dtype=torch.bfloat16) + qkvzba[boundary - 1, 0, :qk_channels] = 10.0 + qkvzba[boundary - 1, 0, k_offset : k_offset + qk_channels] = 10.0 + qkvzba[boundary - 1, 0, v_offset : v_offset + v_channels] = 10.0 + conv_weight = torch.zeros((2 * qk_channels + v_channels, 1, conv_width), device=device) + conv_weight[:qk_channels, 0, conv_width - 2] = 1.0 + conv_weight[k_offset : k_offset + qk_channels, 0, conv_width - 2] = 1.0 + conv_weight[v_offset : v_offset + v_channels, 0, conv_width - 2] = 1.0 + A_log = torch.zeros((num_value_heads,), device=device, dtype=torch.bfloat16) + dt_bias = torch.zeros((num_value_heads,), device=device, dtype=torch.bfloat16) + cu_seqlens = torch.tensor([0, boundary, seq_len], device=device, dtype=torch.int32) + + query, key, value, _, _, _ = fused_streamed_pre_gated_delta_rule( + qkvzba, + conv_weight.to(torch.bfloat16), + None, + A_log, + dt_bias, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + cu_seqlens=cu_seqlens, + ) + + torch.testing.assert_close( + query[0, boundary], + torch.zeros_like(query[0, boundary]), + atol=0.0, + rtol=0.0, + ) + torch.testing.assert_close( + key[0, boundary], + torch.zeros_like(key[0, boundary]), + atol=0.0, + rtol=0.0, + ) + torch.testing.assert_close( + value[0, boundary], + torch.zeros_like(value[0, boundary]), + atol=0.0, + rtol=0.0, + ) + + def test_fused_packed_conv_backward_boundary_isolation(self): + from megatron.core.fusions.fused_pre_gated_delta_rule import ( + fused_streamed_pre_gated_delta_rule, + ) + + seq_len = 5 + boundary = 3 + num_key_heads = 1 + # Keep qkvzba.stride(0) aligned for causal_conv1d's channel-last + # backward guard; the boundary condition under test is independent + # of the value-head repeat factor. + num_value_heads = 4 + key_head_dim = 32 + value_head_dim = 32 + conv_width = 4 + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + v_offset = 2 * qk_channels + k_offset = qk_channels + total_channels = 2 * qk_channels + 2 * v_channels + 2 * num_value_heads + device = torch.cuda.current_device() + + qkvzba = torch.zeros( + (seq_len, 1, total_channels), device=device, dtype=torch.bfloat16, requires_grad=True + ) + conv_weight = torch.zeros( + (2 * qk_channels + v_channels, 1, conv_width), + device=device, + dtype=torch.bfloat16, + requires_grad=True, + ) + with torch.no_grad(): + conv_weight[:qk_channels, 0, conv_width - 2] = 1.0 + conv_weight[k_offset : k_offset + qk_channels, 0, conv_width - 2] = 1.0 + conv_weight[v_offset : v_offset + v_channels, 0, conv_width - 2] = 1.0 + A_log = torch.zeros((num_value_heads,), device=device, dtype=torch.bfloat16, requires_grad=True) + dt_bias = torch.zeros( + (num_value_heads,), device=device, dtype=torch.bfloat16, requires_grad=True + ) + cu_seqlens = torch.tensor([0, boundary, seq_len], device=device, dtype=torch.int32) + + query, key, value, gate, beta, g = fused_streamed_pre_gated_delta_rule( + qkvzba, + conv_weight, + None, + A_log, + dt_bias, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + cu_seqlens=cu_seqlens, + ) + + loss = ( + query[0, boundary].float().sum() + + key[0, boundary].float().sum() + + value[0, boundary].float().sum() + ) + loss = loss + 0.0 * ( + gate.float().sum() + + beta.float().sum() + + g.float().sum() + ) + loss.backward() + leaked_q_grad = qkvzba.grad[boundary - 1, 0, :qk_channels] + leaked_k_grad = qkvzba.grad[boundary - 1, 0, k_offset : k_offset + qk_channels] + leaked_grad = qkvzba.grad[boundary - 1, 0, v_offset : v_offset + v_channels] + torch.testing.assert_close(leaked_q_grad, torch.zeros_like(leaked_q_grad), atol=0.0, rtol=0.0) + torch.testing.assert_close(leaked_k_grad, torch.zeros_like(leaked_k_grad), atol=0.0, rtol=0.0) + torch.testing.assert_close(leaked_grad, torch.zeros_like(leaked_grad), atol=0.0, rtol=0.0) + + @pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.") @pytest.mark.internal class TestGDNCuSeqlensResolve: diff --git a/tests/unit_tests/ssm/test_hybrid_block.py b/tests/unit_tests/ssm/test_hybrid_block.py index 08bf7f2bc28..14caa55aa0a 100644 --- a/tests/unit_tests/ssm/test_hybrid_block.py +++ b/tests/unit_tests/ssm/test_hybrid_block.py @@ -3,7 +3,7 @@ import pytest import torch -from megatron.core.models.hybrid.hybrid_block import HybridStack +from megatron.core.models.hybrid.hybrid_block import HybridStack, HyperConnectionHybridLayer from megatron.core.models.hybrid.hybrid_layer_allocation import Symbols, validate_segment_layers from megatron.core.models.hybrid.hybrid_layer_specs import hybrid_stack_spec from megatron.core.process_groups_config import ProcessGroupCollection @@ -30,8 +30,13 @@ def setup_method(self, method): def get_pg_collection(self): return ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'pp', 'cp']) - def get_mamba_block(self, layer_pattern): + def get_mamba_block(self, layer_pattern, enable_hyper_connections=False): layer_type_list = validate_segment_layers(layer_pattern) + mhc_kwargs = ( + {"enable_hyper_connections": True, "hidden_dropout": 0.0, "mhc_sinkhorn_iterations": 5} + if enable_hyper_connections + else {} + ) transformer_config = TransformerConfig( hidden_size=256, # The Mamba layer places several constraints on this # Need to specify num_attention_heads and num_layers or TransformerConfig @@ -39,6 +44,7 @@ def get_mamba_block(self, layer_pattern): num_layers=len(layer_type_list), num_attention_heads=4, use_cpu_initialization=True, + **mhc_kwargs, ) modules = hybrid_stack_spec.submodules return HybridStack( @@ -49,8 +55,13 @@ def get_mamba_block(self, layer_pattern): pg_collection=self.get_pg_collection(), ) - def get_dsa_mamba_block(self, layer_pattern): + def get_dsa_mamba_block(self, layer_pattern, enable_hyper_connections=False): layer_type_list = validate_segment_layers(layer_pattern) + mhc_kwargs = ( + {"enable_hyper_connections": True, "hidden_dropout": 0.0, "mhc_sinkhorn_iterations": 5} + if enable_hyper_connections + else {} + ) transformer_config = MLATransformerConfig( hidden_size=256, # The Mamba layer places several constraints on this # Need to specify num_attention_heads and num_layers or TransformerConfig @@ -71,6 +82,7 @@ def get_dsa_mamba_block(self, layer_pattern): dsa_indexer_n_heads=8, dsa_indexer_head_dim=64, dsa_indexer_topk=32, + **mhc_kwargs, ) modules = hybrid_stack_spec.submodules return HybridStack( @@ -118,6 +130,161 @@ def test_layer_types(self): assert isinstance(layers[2], TransformerLayer) assert isinstance(layers[2].mlp, MLP) + def test_hyper_connection_layer_wrappers(self): + """mHC wraps each hybrid layer while preserving the layer type underneath.""" + layer_pattern = Symbols.MAMBA + Symbols.ATTENTION + Symbols.MLP + block = self.get_mamba_block(layer_pattern, enable_hyper_connections=True) + layers = block.layers + assert all(isinstance(layer, HyperConnectionHybridLayer) for layer in layers) + assert isinstance(layers[0].inner_layer, MambaLayer) + assert isinstance(layers[1].inner_layer, TransformerLayer) + assert isinstance(layers[1].inner_layer.self_attention, SelfAttention) + assert isinstance(layers[2].inner_layer, TransformerLayer) + assert isinstance(layers[2].inner_layer.mlp, MLP) + + def test_hyper_connection_recompute_plan_for_hybrid_layers(self): + """HybridStack creates per-layer mHC recompute managers when requested.""" + layer_pattern = Symbols.MAMBA + Symbols.ATTENTION + Symbols.MLP + layer_type_list = validate_segment_layers(layer_pattern) + transformer_config = TransformerConfig( + hidden_size=256, + num_layers=len(layer_type_list), + num_attention_heads=4, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + mhc_sinkhorn_iterations=5, + recompute_granularity="selective", + recompute_modules=["core_attn", "mhc"], + ) + block = HybridStack( + transformer_config, + hybrid_stack_spec.submodules, + layer_type_list=layer_type_list, + pp_layer_offset=0, + pg_collection=self.get_pg_collection(), + ) + + managers, block_ends = block._build_mhc_recompute_layer_plan(use_mhc_recompute=True) + assert len(managers) == len(block.layers) + assert all(manager is not None for manager in managers) + assert block_ends[-1] is True + + def test_hyper_connection_gpu_forward(self): + """mHC-enabled HybridStack expands internally and contracts back at the output.""" + layer_pattern = Symbols.MAMBA + Symbols.ATTENTION + Symbols.MLP + block = self.get_mamba_block(layer_pattern, enable_hyper_connections=True) + block.cuda() + micro_batch_size = 2 + sequence_length = 32 + hidden_states = torch.ones((sequence_length, micro_batch_size, block.config.hidden_size)) + hidden_states = hidden_states.cuda() + attention_mask = torch.ones( + (micro_batch_size, 1, sequence_length, sequence_length), dtype=bool + ) + attention_mask = attention_mask.cuda() + output = block(hidden_states, attention_mask=attention_mask) + assert output.shape[0] == sequence_length + assert output.shape[1] == micro_batch_size + assert output.shape[2] == block.config.hidden_size + assert output.dtype == torch.float32 + + def test_hyper_connection_gdn_gpu_forward(self): + """mHC runs through GDN, attention, and Mamba hybrid layers.""" + layer_pattern = Symbols.GDN + Symbols.ATTENTION + Symbols.MAMBA + layer_type_list = validate_segment_layers(layer_pattern) + transformer_config = TransformerConfig( + hidden_size=256, + num_layers=len(layer_type_list), + num_attention_heads=4, + use_cpu_initialization=True, + activation_func=torch.nn.functional.silu, + enable_hyper_connections=True, + hidden_dropout=0.0, + mhc_sinkhorn_iterations=5, + ) + block = HybridStack( + transformer_config, + hybrid_stack_spec.submodules, + layer_type_list=layer_type_list, + pp_layer_offset=0, + pg_collection=self.get_pg_collection(), + ) + block.cuda() + micro_batch_size = 2 + sequence_length = 32 + hidden_states = torch.ones((sequence_length, micro_batch_size, block.config.hidden_size)) + hidden_states = hidden_states.cuda() + attention_mask = torch.ones( + (micro_batch_size, 1, sequence_length, sequence_length), dtype=bool + ).cuda() + output = block(hidden_states, attention_mask=attention_mask) + assert output.shape == (sequence_length, micro_batch_size, block.config.hidden_size) + + def test_hyper_connection_dsa_layer_wrappers(self): + """mHC wraps DeepSeek-style DSA and MLP split layers.""" + layer_pattern = Symbols.MAMBA + Symbols.DS_ATTENTION + Symbols.MLP + block = self.get_dsa_mamba_block(layer_pattern, enable_hyper_connections=True) + layers = block.layers + assert all(isinstance(layer, HyperConnectionHybridLayer) for layer in layers) + assert isinstance(layers[0].inner_layer, MambaLayer) + assert isinstance(layers[1].inner_layer, TransformerLayer) + assert isinstance(layers[1].inner_layer.self_attention, MLASelfAttention) + assert isinstance(layers[1].inner_layer.self_attention.core_attention, DSAttention) + assert isinstance(layers[2].inner_layer, TransformerLayer) + assert isinstance(layers[2].inner_layer.mlp, MLP) + + def test_hyper_connection_pipeline_boundary_shapes(self): + """HybridStack keeps n-stream tensors between PP stages and contracts at the end.""" + layer_type_list = validate_segment_layers(Symbols.MAMBA) + transformer_config = TransformerConfig( + hidden_size=256, + num_layers=len(layer_type_list), + num_attention_heads=4, + use_cpu_initialization=True, + enable_hyper_connections=True, + hidden_dropout=0.0, + mhc_sinkhorn_iterations=5, + ) + modules = hybrid_stack_spec.submodules + first_stage = HybridStack( + transformer_config, + modules, + layer_type_list=layer_type_list, + pp_layer_offset=0, + post_process=False, + pg_collection=self.get_pg_collection(), + ).cuda() + last_stage = HybridStack( + transformer_config, + modules, + pre_process=False, + layer_type_list=layer_type_list, + pp_layer_offset=1, + post_process=True, + pg_collection=self.get_pg_collection(), + ).cuda() + + micro_batch_size = 2 + sequence_length = 32 + hidden_states = torch.ones( + (sequence_length, micro_batch_size, transformer_config.hidden_size), device='cuda' + ) + attention_mask = torch.ones( + (micro_batch_size, 1, sequence_length, sequence_length), dtype=bool, device='cuda' + ) + + pp_hidden = first_stage(hidden_states, attention_mask=attention_mask) + assert pp_hidden.shape == ( + sequence_length, + micro_batch_size, + transformer_config.hidden_size * transformer_config.num_residual_streams, + ) + + last_stage.set_input_tensor(pp_hidden.detach()) + output = last_stage(hidden_states, attention_mask=attention_mask) + assert output.shape == (sequence_length, micro_batch_size, transformer_config.hidden_size) + def test_invalid_layer_types_cause_failure(self): invalid_symbol = '+' assert invalid_symbol not in Symbols.VALID_LAYERS # sanity check. diff --git a/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_csa.py b/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_csa.py index 83c153d698e..9ed44c879e1 100644 --- a/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_csa.py +++ b/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_csa.py @@ -15,6 +15,7 @@ CompressorSubmodules, CSAIndexer, CSAIndexerSubmodules, + _apply_rope, get_compress_topk_idxs, get_window_topk_idxs, unfused_compressed_sparse_attn, @@ -870,3 +871,212 @@ def test_dense_mode_forward_ratio4(self): assert output.shape == (seq_len, batch_size, np_ * hn) assert not torch.isnan(output).any() + + +# =========================================================================== +# _apply_rope tests +# =========================================================================== + + +class TestApplyRope: + """Test ``_apply_rope`` — the layout-aware RoPE wrapper used by + Compressor / CSAIndexer / DSv4HybridAttention. + + Behaviours covered: + + * 3-D ``[seq, batch, head_dim]`` and 4-D ``[seq, batch, heads, head_dim]`` + inputs both work (3-D gets a temporary head-dim unsqueeze). + * Only the trailing ``pos_dim`` components are rotated; the leading + ``nope_dim`` slice is bit-exact unchanged. + * Both ``RotaryEmbedding`` (returns ``Tensor``) and + ``YarnRotaryEmbedding`` (returns ``(emb, mscale)`` tuple) — DSv4 + hybrid silently swaps the class based on ``compress_ratio``. + * Both unfused and fused (``config.apply_rope_fusion=True``) paths + produce the same output (within bf16 precision). + * For ``ratio > 1`` the rotary table is built at + ``rotary_seq_len * ratio`` and strided by ``ratio``. + """ + + @pytest.fixture(scope='class', autouse=True) + def setup_method(self, request): + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, pipeline_model_parallel_size=1 + ) + torch.manual_seed(0) + model_parallel_cuda_manual_seed(0) + cls = request.cls + cls.pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp']) + # head_dim 32 = nope 24 + pos 8 + cls.config = _make_mla_config(v_head_dim=32, qk_pos_emb_head_dim=8) + yield + Utils.destroy_model_parallel() + + def _make_rotary(self, kind: str): + from megatron.core.models.common.embeddings import RotaryEmbedding, YarnRotaryEmbedding + + pos_dim = self.config.qk_pos_emb_head_dim + if kind == 'rope': + return RotaryEmbedding( + pos_dim, rotary_percent=1.0, rotary_base=10000, cp_group=self.pg_collection.cp + ) + if kind == 'yarn': + return YarnRotaryEmbedding( + pos_dim, + rotary_base=40000, + scaling_factor=40, + original_max_position_embeddings=4096, + beta_fast=32, + beta_slow=1, + mscale=1.0, + mscale_all_dim=0.0, + cp_group=self.pg_collection.cp, + ) + raise ValueError(kind) + + def _config_with(self, *, apply_rope_fusion: bool): + # Reuse the class-level config; only flip the fusion flag. + cfg = self.config + cfg.apply_rope_fusion = apply_rope_fusion + return cfg + + _ROTARY_FUSION_COMBOS = [ + pytest.param('rope', False, id='rope-unfused'), + pytest.param('rope', True, id='rope-fused'), + pytest.param('yarn', False, id='yarn-unfused'), + pytest.param('yarn', True, id='yarn-fused'), + ] + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize(("rotary_kind", "apply_rope_fusion"), _ROTARY_FUSION_COMBOS) + @pytest.mark.parametrize("input_ndim", [3, 4], ids=['3d', '4d']) + @pytest.mark.parametrize("ratio", [1, 4], ids=['ratio_1', 'ratio_4']) + def test_apply_rope(self, rotary_kind, apply_rope_fusion, input_ndim, ratio): + """Output shape == input shape; no NaN; nope-dim slice is + bit-exact unchanged. Sweeps the valid combinations of rotary + class × apply_rope_fusion × input rank × ratio. Yarn's + tuple-return is covered by the ``'yarn-*'`` combos. + """ + rotary = self._make_rotary(rotary_kind).cuda() + nope = self.config.v_head_dim - self.config.qk_pos_emb_head_dim + pos = self.config.qk_pos_emb_head_dim + head_dim = nope + pos + seq, batch, heads = 8, 2, 4 + cfg = self._config_with(apply_rope_fusion=apply_rope_fusion) + + shape = (seq, batch, head_dim) if input_ndim == 3 else (seq, batch, heads, head_dim) + x = torch.randn(*shape, dtype=torch.bfloat16, device='cuda') + # ``fused_mla_rope_inplace`` mutates the input — give it a copy so + # the nope-dim equality check below still has the original. + out = _apply_rope( + x.clone(), + nope, + pos, + rotary, + cfg, + rotary_seq_len=seq, + ratio=ratio, + cp_group=self.pg_collection.cp, + ) + + assert out.shape == x.shape + assert out.dtype == x.dtype + assert not torch.isnan(out).any() + # The leading nope_dim slice is the identity portion of RoPE. + assert torch.equal( + out[..., :nope], x[..., :nope] + ), "RoPE must not touch the first nope_dim components" + # Trailing pos_dim should rotate at non-zero positions. + pe_changed = (out[..., nope:] != x[..., nope:]).any(dim=-1).flatten() + assert pe_changed[ + 1: + ].any(), "RoPE should rotate the trailing pos_dim components for seq > 0" + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize("rotary_kind", ['rope', 'yarn']) + def test_3d_input_matches_4d_with_single_head(self, rotary_kind): + """For a single-head input, the 3-D ``(s, b, d)`` and 4-D + ``(s, b, 1, d)`` invocations must produce numerically identical + output (3-D path just inserts a temporary head dim). + """ + rotary = self._make_rotary(rotary_kind).cuda() + nope = self.config.v_head_dim - self.config.qk_pos_emb_head_dim + pos = self.config.qk_pos_emb_head_dim + head_dim = nope + pos + seq, batch = 8, 2 + cfg = self._config_with(apply_rope_fusion=False) + + x_3d = torch.randn(seq, batch, head_dim, dtype=torch.bfloat16, device='cuda') + x_4d = x_3d.unsqueeze(-2) + + out_3d = _apply_rope( + x_3d, + nope, + pos, + rotary, + cfg, + rotary_seq_len=seq, + ratio=1, + cp_group=self.pg_collection.cp, + ) + out_4d = _apply_rope( + x_4d, + nope, + pos, + rotary, + cfg, + rotary_seq_len=seq, + ratio=1, + cp_group=self.pg_collection.cp, + ) + + assert out_3d.shape == x_3d.shape + assert out_4d.shape == x_4d.shape + assert torch.equal(out_3d, out_4d.squeeze(-2)) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize("rotary_kind", ['rope', 'yarn']) + def test_ratio_strides_rotary_table(self, rotary_kind): + """For ``ratio > 1``, the rotary table is built at + ``rotary_seq_len * ratio`` and strided by ``ratio``. The result + with ``ratio=k`` must equal an ``apply_rope`` call on the same + positions of a length-``rotary_seq_len * k`` table. + """ + rotary = self._make_rotary(rotary_kind).cuda() + nope = self.config.v_head_dim - self.config.qk_pos_emb_head_dim + pos = self.config.qk_pos_emb_head_dim + head_dim = nope + pos + seq, batch, heads, ratio = 4, 1, 2, 4 + cfg = self._config_with(apply_rope_fusion=False) + + x_comp = torch.randn(seq, batch, heads, head_dim, dtype=torch.bfloat16, device='cuda') + out_comp = _apply_rope( + x_comp.clone(), + nope, + pos, + rotary, + cfg, + rotary_seq_len=seq, + ratio=ratio, + cp_group=self.pg_collection.cp, + ) + + x_full = torch.zeros( + seq * ratio, batch, heads, head_dim, dtype=torch.bfloat16, device='cuda' + ) + x_full[::ratio][:seq] = x_comp + out_full = _apply_rope( + x_full, + nope, + pos, + rotary, + cfg, + rotary_seq_len=seq * ratio, + ratio=1, + cp_group=self.pg_collection.cp, + ) + out_ref = out_full[::ratio][:seq] + + assert torch.allclose(out_comp, out_ref, rtol=1e-3, atol=1e-3), ( + f"ratio={ratio} stride mismatch: " + f"max abs diff = {(out_comp - out_ref).abs().max().item():.3e}" + ) diff --git a/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_dsa.py b/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_dsa.py index 757b9dd283a..1e8b1b454ef 100644 --- a/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_dsa.py +++ b/tests/unit_tests/transformer/experimental_attention_variant/test_attention_variant_dsa.py @@ -212,6 +212,54 @@ def test_dsa_indexer_loss_sparse(self, seqlen_and_topk): assert loss_sparse >= 0 assert loss_dense >= 0 + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_dsa_indexer_loss_per_token_scale(self, seqlen_and_topk): + batch_size = 2 + seqlen = seqlen_and_topk[0] + num_heads = 4 + head_dim = 128 + index_topk = seqlen_and_topk[1] + + index_scores = torch.randn(batch_size, seqlen, seqlen, dtype=torch.float32).cuda() + causal_mask = torch.triu( + torch.full( + (seqlen, seqlen), float('-inf'), dtype=torch.float32, device=index_scores.device + ), + diagonal=1, + ) + masked_index_scores = index_scores + causal_mask + topk_k = min(index_topk, seqlen) + topk_indices = masked_index_scores.topk(topk_k, dim=-1)[1] + + query = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda() + key = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda() + softmax_scale = head_dim**-0.5 + + for sparse_loss in [False, True]: + loss_mean = compute_dsa_indexer_loss( + index_scores=index_scores.clone(), + topk_indices=topk_indices, + query=query, + key=key, + softmax_scale=softmax_scale, + loss_coeff=1.0, + sparse_loss=sparse_loss, + pg_collection=self.pg_collection, + ) + loss_sum = compute_dsa_indexer_loss( + index_scores=index_scores.clone(), + topk_indices=topk_indices, + query=query, + key=key, + softmax_scale=softmax_scale, + loss_coeff=1.0, + sparse_loss=sparse_loss, + pg_collection=self.pg_collection, + calculate_per_token_loss=True, + ) + + assert torch.allclose(loss_sum, loss_mean * (batch_size * seqlen), rtol=1e-3, atol=1e-3) + class TestDSAIndexerLossAutoScaler: """Test DSAIndexerLossAutoScaler autograd function.""" @@ -274,6 +322,8 @@ def test_backward_pass(self): atol=0, ), f"Gradient should be scaled by loss scale, expected {expected_grad_per_element}, got {dummy_input.grad[0].item()}" + DSAIndexerLossAutoScaler.main_loss_backward_scale = None + class TestFusedDSAIndexerLossGradient: """Test that FusedDSAIndexerLoss manual backward matches autograd backward.""" @@ -289,8 +339,10 @@ def setup_method(self, request): yield Utils.destroy_model_parallel() + @pytest.mark.flaky_in_dev @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") - def test_fused_indexer_loss_gradient_matches_autograd(self): + @pytest.mark.parametrize("calculate_per_token_loss", [False, True]) + def test_fused_indexer_loss_gradient_matches_autograd(self, calculate_per_token_loss): """ Test that the manually written backward in FusedDSAIndexerLoss produces the same gradients as PyTorch autograd on the unfused implementation. @@ -305,7 +357,10 @@ def test_fused_indexer_loss_gradient_matches_autograd(self): for seqlen, index_topk in [[16, 8], [32, 16], [64, 32]]: for sparse_loss in [False, True]: - tag = f"[seqlen={seqlen}, topk={index_topk}, sparse={sparse_loss}]" + tag = ( + f"[seqlen={seqlen}, topk={index_topk}, sparse={sparse_loss}, " + f"per_token={calculate_per_token_loss}]" + ) torch.manual_seed(42) q_ref = ( @@ -351,6 +406,7 @@ def test_fused_indexer_loss_gradient_matches_autograd(self): loss_coeff=loss_coeff, sparse_loss=sparse_loss, pg_collection=self.pg_collection, + calculate_per_token_loss=calculate_per_token_loss, ) loss_ref.backward() @@ -375,6 +431,7 @@ def test_fused_indexer_loss_gradient_matches_autograd(self): mask, sparse_loss, self.pg_collection, + calculate_per_token_loss, ) loss_fused.backward() @@ -472,6 +529,7 @@ def test_fused_indexer_loss_gradient_tp_consistency(self): mask, sparse_loss, pg_collection_tp1, + False, ) loss_tp1.backward() @@ -528,6 +586,7 @@ def test_fused_indexer_loss_gradient_tp_consistency(self): mask, sparse_loss, pg_collection_tpn, + False, ) loss_tpn.backward() diff --git a/tests/unit_tests/transformer/experimental_attention_variant/test_dsa_kernels.py b/tests/unit_tests/transformer/experimental_attention_variant/test_dsa_kernels.py new file mode 100644 index 00000000000..a19a0e35e4b --- /dev/null +++ b/tests/unit_tests/transformer/experimental_attention_variant/test_dsa_kernels.py @@ -0,0 +1,2591 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Unit tests for ``megatron.core.transformer.experimental_attention_variant.dsa_kernels``. + +Coverage: + +* Pure-Python helpers: :func:`local_to_global_flat`, :func:`build_flat_topk_idxs`, + :func:`_kl_loss_from_target_predict` — full correctness checks; no GPU + kernels required (CPU is fine). +* Lazy-import gates: :func:`_ensure_flash_mla`, :func:`_ensure_dsa_namespace` + raise informative ``ImportError`` when the optional packages are missing. +* GPU helpers: :func:`_get_topk_alignment` — runs only on CUDA. +* Wrapper functions :func:`_dsa_fwd_flash_mla`, :func:`indexer_topk`, + :func:`dsa_sparse_attn`, :func:`fused_indexer_sparse_attn` — exercised with + ``unittest.mock`` stand-ins for the underlying ``flash_mla`` / + ``cudnn.DSA`` kernels so the data-marshalling logic (shape conversions, + TopK padding, predict/target/KL composition, autograd plumbing) is + validated without requiring the real CUDA kernels. +""" + +from __future__ import annotations + +import math +import sys +import types +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from megatron.core.transformer.experimental_attention_variant import dsa_kernels as dk +from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + FusedIndexerSparseAttnFunc, + SparseAttnFunc, + _dsa_fwd_flash_mla, + _ensure_dsa_namespace, + _ensure_flash_mla, + _get_topk_alignment, + _kl_loss_from_dense_scores, + _kl_loss_from_target_predict, + build_flat_topk_idxs, + dsa_sparse_attn, + fused_indexer_sparse_attn, + indexer_topk, + local_to_global_flat, +) + +# --------------------------------------------------------------------------- +# Test fixtures / helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def reset_lazy_kernel_state(): + """Reset the module-level lazy import slots before/after each test. + + The wrapper-function tests patch ``_flash_mla_sparse_fwd`` / ``_DSA`` + directly; we need to ensure each test starts from a clean slate so the + lazy ``_ensure_*`` calls are exercised consistently. + """ + saved_flash = dk._flash_mla_sparse_fwd + saved_dsa = dk._DSA + dk._flash_mla_sparse_fwd = None + dk._DSA = None + yield + dk._flash_mla_sparse_fwd = saved_flash + dk._DSA = saved_dsa + + +def _make_local_idxs(b: int, sq: int, topk: int, *, with_invalid: bool = False) -> torch.Tensor: + """Build a deterministic ``(b, sq, topk)`` int64 tensor of local indices. + + Values for batch ``i``, query ``s``, slot ``k`` are + ``i * 100 + s * 10 + k``. When ``with_invalid`` is True every other + slot is replaced with -1. + """ + base = ( + torch.arange(b, dtype=torch.int64).view(b, 1, 1) * 100 + + torch.arange(sq, dtype=torch.int64).view(1, sq, 1) * 10 + + torch.arange(topk, dtype=torch.int64).view(1, 1, topk) + ) + if with_invalid: + mask = torch.arange(topk).view(1, 1, topk) % 2 == 1 + base = torch.where(mask.expand(b, sq, topk), torch.full_like(base, -1), base) + return base + + +def _uniform_dist(B, S, K, dev): + """Uniform ``1/K`` distribution of shape ``(B, S, K)``.""" + return torch.full((B, S, K), 1.0 / max(K, 1), dtype=torch.float32, device=dev) + + +def _peaked_dist(B, S, K, dev, peak_idx=0): + """Distribution with all probability mass on ``peak_idx``.""" + out = torch.zeros(B, S, K, dtype=torch.float32, device=dev) + out[..., peak_idx] = 1.0 + return out + + +# --------------------------------------------------------------------------- +# local_to_global_flat +# --------------------------------------------------------------------------- + + +class TestLocalToGlobalFlat: + """Pure-Python index conversion (no GPU required).""" + + @pytest.mark.parametrize( + "b, sq, topk, with_invalid", + [ + (1, 4, 5, False), # b=1 identity case + (2, 3, 4, False), # basic multi-batch + (3, 5, 4, False), # larger batch (stresses the formula) + (2, 3, 6, True), # invalid entries interleaved with valid ones + ], + ids=['b1_identity', 'basic', 'larger_b', 'with_invalid'], + ) + def test_global_index_conversion(self, b, sq, topk, with_invalid): + """Shape, dtype, ``-1`` preservation, and the formula + ``global[s*b + bid, k] = local[bid, s, k] * b + bid`` (for valid entries) + in one fixture. Row ``r`` of the output corresponds to query ``s = r // b`` + and batch id ``bid = r % b``. + """ + local = _make_local_idxs(b, sq, topk, with_invalid=with_invalid) + out = local_to_global_flat(local, b, seqlen_kv=128) + + assert out.shape == (sq * b, topk) + assert out.dtype == torch.int32 + + permuted = local.permute(1, 0, 2).reshape(sq * b, topk) + batch_ids = (torch.arange(sq * b) % b).unsqueeze(1) + expected = torch.where( + permuted >= 0, permuted * b + batch_ids, torch.full_like(permuted, -1) + ).int() + assert torch.equal(out, expected) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_cpu_cuda_parity(self): + """CPU and CUDA execution paths produce identical results.""" + local = _make_local_idxs(b=2, sq=4, topk=3, with_invalid=True) + out_cpu = local_to_global_flat(local, 2, seqlen_kv=64) + out_cuda = local_to_global_flat(local.cuda(), 2, seqlen_kv=64) + assert torch.equal(out_cpu, out_cuda.cpu()) + + +# --------------------------------------------------------------------------- +# build_flat_topk_idxs +# --------------------------------------------------------------------------- + + +class TestBuildFlatTopkIdxs: + """Pure-Python multi-group concat + optional compaction.""" + + @pytest.mark.parametrize( + "group_specs", + [ + # Each spec is a list of (topk_i, with_invalid) for each group. + [(4, False)], # single group, all valid + [(2, False), (3, False)], # two groups, all valid + ], + ids=['single_group', 'two_groups'], + ) + def test_non_compact_concat_then_globalise(self, group_specs): + """Without ``compact`` the helper concatenates groups along ``topk`` + and applies the local→global conversion verbatim. + """ + b, sq = 2, 3 + groups = [ + _make_local_idxs(b, sq, t, with_invalid=inv) + 50 * i + for i, (t, inv) in enumerate(group_specs) + ] + total_topk = sum(t for t, _ in group_specs) + + flat, length = build_flat_topk_idxs(*groups, batch_size=b, seqlen_kv=256) + + expected = local_to_global_flat(torch.cat(groups, dim=-1), b, seqlen_kv=256) + assert flat.shape == (sq * b, total_topk) + assert flat.dtype == torch.int32 + assert torch.equal(flat, expected) + assert length is None + + @pytest.mark.parametrize( + "group_specs, expected_valid_per_row", + [ + # Single group: 6 slots with every odd slot invalid → 3 valid. + ([(6, True)], 3), + # Two groups: g1 has 2 valid out of 4, g2 fully valid (2) → 4 valid. + ([(4, True), (2, False)], 4), + ], + ids=['single_group', 'two_groups'], + ) + def test_compact_packs_valid_first(self, group_specs, expected_valid_per_row): + """With ``compact=True`` the helper packs valid entries to the front + of each row, fills the tail with ``-1``, and returns a per-row + ``topk_length`` that equals the count of valid entries. + """ + b, sq = 2, 3 + groups = [ + _make_local_idxs(b, sq, t, with_invalid=inv) + 100 * i + for i, (t, inv) in enumerate(group_specs) + ] + total_topk = sum(t for t, _ in group_specs) + + flat, length = build_flat_topk_idxs(*groups, batch_size=b, seqlen_kv=512, compact=True) + + assert flat.shape == (sq * b, total_topk) + assert flat.dtype == torch.int32 + assert length is not None + assert length.shape == (sq * b,) + assert length.dtype == torch.int32 + + # Per-row layout: valid global indices first, then -1 padding. + for row in range(sq * b): + n = int(length[row]) + assert n == expected_valid_per_row, f"row {row}: wrong length" + assert torch.all(flat[row, :n] >= 0), f"row {row}: leading entries should be valid" + assert torch.all(flat[row, n:] == -1), f"row {row}: trailing entries should be -1" + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_compact_cuda_path(self, reset_lazy_kernel_state): + """Combined coverage for the CUDA compact path (sub-blocks + self-label on failure): + + * (a) Dispatch + plumbing (mocked compactify): the wrapper is + called exactly once, with the already-globalised + ``(sq*b, total_topk)`` int32 tensor as input, and its + returned ``(indices, topk_length)`` flow back verbatim. + * (b) End-to-end parity (real cuDNN, skipped without it): the + cuDNN ``compactify`` kernel produces the same ``(flat, + length)`` pair as the pure-PyTorch CPU fallback. + """ + # ---- (a) dispatch via mocked compactify -------------------------- + b, sq, topk = 2, 3, 4 + local = _make_local_idxs(b, sq, topk, with_invalid=True).to(torch.int32, copy=False).cuda() + compact_indices = torch.full((sq * b, topk), 99, dtype=torch.int32, device='cuda') + topk_length = torch.full((sq * b,), 7, dtype=torch.int32, device='cuda') + + captured = {} + + def fake_compactify(global_idxs): + captured['input'] = global_idxs + return {'indices': compact_indices, 'topk_length': topk_length} + + fake_dsa = MagicMock(name='_DSA_compactify_stub') + fake_dsa.compactify_wrapper.side_effect = fake_compactify + dk._DSA = fake_dsa + + flat, length = build_flat_topk_idxs(local, batch_size=b, seqlen_kv=512, compact=True) + fake_dsa.compactify_wrapper.assert_called_once() + kernel_input = captured['input'] + assert kernel_input.shape == (sq * b, topk), "(a) wrapper input shape" + assert kernel_input.dtype == torch.int32, "(a) wrapper input dtype" + assert kernel_input.is_cuda, "(a) wrapper input not on CUDA" + expected_input = local_to_global_flat(local, b, seqlen_kv=512) + assert torch.equal( + kernel_input, expected_input + ), "(a) wrapper input != local_to_global_flat(local)" + assert flat is compact_indices, "(a) returned flat is not the kernel output" + assert length is topk_length, "(a) returned length is not the kernel output" + + # ---- (b) real-kernel parity vs CPU fallback ---------------------- + # Skipped when cuDNN is not installed; reset state so the real + # _DSA import happens on the next call inside build_flat_topk_idxs. + try: + cudnn = pytest.importorskip("cudnn") + except pytest.skip.Exception: + return # already passed (a); skip the parity sub-block silently + if not hasattr(cudnn, 'DSA'): + return + dk._DSA = None # force real lazy-import + + b2, sq2 = 4, 5 + local_a = _make_local_idxs(b2, sq2, 6, with_invalid=True) + local_b = _make_local_idxs(b2, sq2, 4, with_invalid=False) + 200 + + flat_cpu, len_cpu = build_flat_topk_idxs( + local_a, local_b, batch_size=b2, seqlen_kv=512, compact=True + ) + flat_cuda, len_cuda = build_flat_topk_idxs( + local_a.cuda(), local_b.cuda(), batch_size=b2, seqlen_kv=512, compact=True + ) + assert torch.equal( + flat_cpu, flat_cuda.cpu() + ), "(b) flat tensor differs between CPU fallback and cuDNN kernel" + assert torch.equal( + len_cpu, len_cuda.cpu() + ), "(b) length tensor differs between CPU fallback and cuDNN kernel" + + +# --------------------------------------------------------------------------- +# _kl_loss_from_target_predict +# --------------------------------------------------------------------------- + + +class TestKLLossFromTargetPredict: + """Pure-Python KL loss computation: combined assertions for all + properties (scalar/dtype, identity, non-negativity, coeff linearity, + invalid-row masking, analytical formula).""" + + def test_kl_loss_properties(self): + """All KL-loss invariants checked sequentially. Each block raises + an informative ``AssertionError`` so a failure pinpoints the + broken sub-property. + """ + torch.manual_seed(0) + b, sq, topk = 2, 3, 4 + topk_indices = torch.zeros(b, sq, topk, dtype=torch.int32) + + # ---- (a) scalar/dtype + identity: KL(p || p) == 0 ----------------- + identical = torch.softmax(torch.randn(b, sq, topk), dim=-1) + loss_identical = _kl_loss_from_target_predict( + identical, identical.clone(), topk_indices, loss_coeff=1.0 + ) + assert loss_identical.shape == torch.Size([]), "identity: not scalar" + assert loss_identical.dtype == torch.float32, "identity: not fp32" + assert torch.allclose( + loss_identical, torch.tensor(0.0), atol=1e-6 + ), f"identity: KL(p || p) != 0 (got {loss_identical.item()})" + + # ---- (b) non-negativity + linearity in loss_coeff ----------------- + target = torch.softmax(torch.randn(b, sq, topk), dim=-1) + predict = torch.softmax(torch.randn(b, sq, topk), dim=-1) + loss_1 = _kl_loss_from_target_predict(target, predict, topk_indices, loss_coeff=1.0) + loss_3 = _kl_loss_from_target_predict(target, predict, topk_indices, loss_coeff=3.0) + assert loss_1.item() >= 0.0, f"non-negativity: got {loss_1.item()}" + assert torch.allclose( + loss_3, 3.0 * loss_1, atol=1e-5, rtol=1e-5 + ), f"linearity: 3*loss_1 = {3*loss_1.item()} vs loss_3 = {loss_3.item()}" + + # ---- (c) invalid-row masking -------------------------------------- + # Construct deterministic distributions with strictly-positive per-row KL. + t_inv = torch.full((b, sq, topk), 0.1, dtype=torch.float32) + t_inv[..., 0] = 0.7 + p_inv = torch.full((b, sq, topk), 0.7, dtype=torch.float32) / topk + p_inv[..., -1] = 1.0 - p_inv[..., :-1].sum(dim=-1) + + idx_all_valid = torch.zeros(b, sq, topk, dtype=torch.int32) + loss_full = _kl_loss_from_target_predict(t_inv, p_inv, idx_all_valid, loss_coeff=1.0) + assert loss_full.item() > 0, "all-valid baseline must be positive" + + # Mark the first row of every batch invalid → fewer valid rows, + # smaller KL sum, same denominator (mean over all (B, S_q)). + idx_partial = idx_all_valid.clone() + idx_partial[:, 0, :] = -1 + loss_partial = _kl_loss_from_target_predict(t_inv, p_inv, idx_partial, loss_coeff=1.0) + assert ( + loss_partial.item() < loss_full.item() + ), f"partial-invalid: {loss_partial.item()} should be < {loss_full.item()}" + + # All-invalid → loss exactly 0. + idx_all_invalid = torch.full_like(idx_all_valid, -1) + loss_zero = _kl_loss_from_target_predict(t_inv, p_inv, idx_all_invalid, loss_coeff=1.0) + assert loss_zero.item() == 0.0, f"all-invalid: got {loss_zero.item()}" + + # ---- (d) analytical formula: target = δ_0, predict = uniform(1/K) - + # per-row KL = log(K); mean = log(K); loss = coeff * log(K). + target_d = _peaked_dist(b, sq, topk, 'cpu', peak_idx=0) + predict_d = torch.full((b, sq, topk), 1.0 / topk, dtype=torch.float32) + loss_d = _kl_loss_from_target_predict(target_d, predict_d, topk_indices, loss_coeff=2.5) + expected = 2.5 * math.log(topk) + assert torch.allclose( + loss_d, torch.tensor(expected), rtol=1e-5, atol=1e-5 + ), f"analytical: {loss_d.item()} vs expected {expected}" + + def test_per_token_loss_reports_raw_sum(self): + torch.manual_seed(1) + b, sq, topk = 2, 5, 4 + target = torch.softmax(torch.randn(b, sq, topk), dim=-1) + predict = torch.softmax(torch.randn(b, sq, topk), dim=-1) + topk_indices = torch.zeros(b, sq, topk, dtype=torch.int32) + + loss_mean = _kl_loss_from_target_predict(target, predict, topk_indices, loss_coeff=0.5) + loss_sum = _kl_loss_from_target_predict( + target, predict, topk_indices, loss_coeff=0.5, calculate_per_token_loss=True + ) + + assert torch.allclose(loss_sum, loss_mean * (b * sq), rtol=1e-5, atol=1e-5) + + +class TestKLLossFromDenseScores: + def test_per_token_loss_reports_raw_sum(self): + b, sq, sk = 2, 5, 4 + loss_coeff = 0.5 + + attn_score = _peaked_dist(b, sq, sk, 'cpu', peak_idx=0) + attn_l1norm = torch.ones(b, sq, dtype=torch.float32) + index_score = torch.zeros(b, sq, sk, dtype=torch.float32) + index_lse = torch.full((b, sq), math.log(sk), dtype=torch.float32) + + loss_mean = _kl_loss_from_dense_scores( + attn_score, attn_l1norm, index_score, index_lse, loss_coeff + ) + loss_sum = _kl_loss_from_dense_scores( + attn_score, + attn_l1norm, + index_score, + index_lse, + loss_coeff, + calculate_per_token_loss=True, + ) + + assert torch.allclose(loss_sum, loss_mean * (b * sq), rtol=1e-5, atol=1e-5) + + +# --------------------------------------------------------------------------- +# _ensure_flash_mla / _ensure_dsa_namespace +# --------------------------------------------------------------------------- + + +_LAZY_IMPORT_CASES = [ + pytest.param( + 'flash_mla', + 'flash_mla_sparse_fwd', + _ensure_flash_mla, + '_flash_mla_sparse_fwd', + "FlashMLA is required", + id='flash_mla', + ), + pytest.param( + 'cudnn', 'DSA', _ensure_dsa_namespace, '_DSA', "cudnn-frontend DSA", id='cudnn_dsa' + ), +] + + +@pytest.mark.parametrize( + "module_name, attr_name, ensure_fn, slot_name, error_match", _LAZY_IMPORT_CASES +) +class TestLazyKernelImports: + """Lazy-import behaviour shared by ``_ensure_flash_mla`` and + ``_ensure_dsa_namespace``: combined error-on-missing + caches-on-success + fixture (assertion blocks self-label on failure). + """ + + def test_lazy_import_raises_and_caches( + self, reset_lazy_kernel_state, module_name, attr_name, ensure_fn, slot_name, error_match + ): + # ---- (a) raises informative ImportError when the module is absent -- + # Setting ``sys.modules[name] = None`` makes ``import name`` fail. + with patch.dict(sys.modules, {module_name: None}): + with pytest.raises(ImportError, match=error_match): + ensure_fn() + + # ---- (b) caches the import on success ------------------------------ + sentinel = MagicMock(name=f"{attr_name}_sentinel") + fake_module = types.ModuleType(module_name) + setattr(fake_module, attr_name, sentinel) + + with patch.dict(sys.modules, {module_name: fake_module}): + ensure_fn() + assert ( + getattr(dk, slot_name) is sentinel + ), f"(b) {module_name}: ensure_fn() did not bind sentinel" + + # Second call must be a no-op — even after the sys.modules entry is gone. + with patch.dict(sys.modules, {}, clear=False): + sys.modules.pop(module_name, None) + ensure_fn() + assert ( + getattr(dk, slot_name) is sentinel + ), f"(b) {module_name}: cached sentinel was lost on 2nd call" + + +# --------------------------------------------------------------------------- +# _get_topk_alignment +# --------------------------------------------------------------------------- + + +class TestGetTopkAlignment: + """Architecture-dependent alignment for FlashMLA top-K padding.""" + + @pytest.mark.parametrize( + "sm_major, expected", [(7, 128), (8, 128), (9, 128), (10, 64), (12, 64), (13, 64)] + ) + def test_alignment_per_sm(self, sm_major, expected): + """SM10x and newer use 64-byte TopK alignment; older arches use 128.""" + with patch('torch.cuda.get_device_capability', return_value=(sm_major, 0)): + assert _get_topk_alignment() == expected + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_runs_on_real_gpu(self): + """On any real GPU the alignment must agree with the documented rule.""" + align = _get_topk_alignment() + sm = torch.cuda.get_device_capability() + expected = 64 if sm[0] >= 10 else 128 + assert align == expected + + +# --------------------------------------------------------------------------- +# _dsa_fwd_flash_mla — wrapper around flash_mla.flash_mla_sparse_fwd +# --------------------------------------------------------------------------- + + +def _make_flash_mla_stub(d_v: int = 512, *, lse_scalar: float = 0.0, out_fill: float = 0.0): + """Build a callable stand-in for ``flash_mla.flash_mla_sparse_fwd``. + + The real kernel signature is + ``(q, kv, indices, softmax_scale, d_v, attn_sink, topk_length, indexer_topk)`` + and returns ``(out, max_logits, lse)`` or ``(out, max_logits, lse, lse_indexer)`` + when ``indexer_topk > 0``. + + The stub returns deterministic, easily-distinguishable tensors so callers + can numerically verify the wrapper's reshape / split logic. The most + recent ``out`` and ``lse`` are stashed on ``stub.last_out`` / + ``stub.last_lse`` for direct equality checks. + """ + + stub = MagicMock(name='flash_mla_sparse_fwd_stub') + + def _impl(q, kv, indices, softmax_scale, d_v, attn_sink, topk_length, indexer_topk): + total_S_q, H, _D = q.shape + # Distinguishable per-element pattern: out[i, h, k] = out_fill + i + 0.001*h + 1e-6*k + # (works in bf16 at this magnitude, useful for verifying that the + # wrapper does not silently reshape across the wrong axes). + idx_i = torch.arange(total_S_q, dtype=torch.float32, device=q.device).view(-1, 1, 1) + idx_h = torch.arange(H, dtype=torch.float32, device=q.device).view(1, -1, 1) + idx_k = torch.arange(d_v, dtype=torch.float32, device=q.device).view(1, 1, -1) + out_f32 = out_fill + idx_i + 0.001 * idx_h + 1e-6 * idx_k + out = out_f32.to(q.dtype) + max_logits = torch.zeros(total_S_q, H, dtype=torch.float32, device=q.device) + # lse[i, h] = lse_scalar + i + 0.5*h — a deterministic pattern. + lse = lse_scalar + ( + torch.arange(total_S_q, dtype=torch.float32, device=q.device).view(-1, 1) + + 0.5 * torch.arange(H, dtype=torch.float32, device=q.device).view(1, -1) + ) + stub.last_out = out + stub.last_lse = lse + if indexer_topk > 0: + # Make lse_indexer distinct from lse so we can tell which one the + # wrapper returned. + lse_indexer = lse + 100.0 + stub.last_lse_indexer = lse_indexer + return out, max_logits, lse, lse_indexer + stub.last_lse_indexer = None + return out, max_logits, lse + + stub.side_effect = _impl + stub.last_out = None + stub.last_lse = None + stub.last_lse_indexer = None + return stub + + +class TestDsaFwdFlashMla: + """Adapter logic around FlashMLA: shape massaging, TopK padding, return + tuples — including numerical pass-through of the kernel outputs. + """ + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_dsa_fwd_flash_mla_adapter(self, reset_lazy_kernel_state): + """All adapter behaviours in one fixture (assertion blocks self-label + on failure): + + * (a) TopK is padded up to GPU-specific alignment; padded slots are + ``-1``; ``out`` / ``lse`` are passed through verbatim; + kernel arg shapes match the SBHD-flat → MQA-h_kv=1 contract. + * (b) ``indexer_topk == 0`` -> ``lse_indexer is None``. + * (c) ``0 < indexer_topk < TopK`` -> kernel's ``lse_indexer`` is + returned verbatim (not silently swapped for ``lse``). + * (d) ``indexer_topk == TopK`` -> fallback to ``lse.clone()`` (a + kernel snapshot quirk). + * (e) ``indexer_topk > 0 + topk_length`` is rejected with the + expected error. + """ + total_sq, H, D = 4, 2, 512 + align = _get_topk_alignment() + + q = torch.randn(total_sq, H, D, dtype=torch.bfloat16, device='cuda') + kv = torch.randn(8, D, dtype=torch.bfloat16, device='cuda') + + # ---- (a) padding + numerical pass-through ------------------------ + TopK_unpadded = 5 + expected_padded = ((TopK_unpadded + align - 1) // align) * align + topk_idxs = torch.arange(total_sq * TopK_unpadded, dtype=torch.int32, device='cuda').view( + total_sq, TopK_unpadded + ) + + stub = _make_flash_mla_stub(d_v=D) + dk._flash_mla_sparse_fwd = stub + + out, lse, lse_indexer = _dsa_fwd_flash_mla(q, kv, topk_idxs, softmax_scale=0.5, d_v=D) + assert lse_indexer is None, "(a) lse_indexer should be None when indexer_topk=0" + assert torch.equal(out, stub.last_out), "(a) out is not pass-through" + assert torch.equal(lse, stub.last_lse), "(a) lse is not pass-through" + + called_args = stub.call_args.args + kv_3d, indices_arg = called_args[1], called_args[2] + assert kv_3d.shape == (8, 1, D), f"(a) KV shape {tuple(kv_3d.shape)} != (8, 1, {D})" + assert indices_arg.shape == (total_sq, 1, expected_padded), ( + f"(a) indices shape {tuple(indices_arg.shape)} != " + f"({total_sq}, 1, {expected_padded})" + ) + if expected_padded > TopK_unpadded: + assert torch.all( + indices_arg[..., TopK_unpadded:] == -1 + ), "(a) padded slots should be -1" + assert torch.equal( + indices_arg[..., :TopK_unpadded].squeeze(1), topk_idxs + ), "(a) original entries should survive padding unchanged" + + # ---- (b–d) indexer_topk branches --------------------------------- + TopK = align # already aligned, no padding + topk_idxs_aligned = torch.zeros(total_sq, TopK, dtype=torch.int32, device='cuda') + + stub = _make_flash_mla_stub(d_v=D, lse_scalar=1.5) + dk._flash_mla_sparse_fwd = stub + + # (b) indexer_topk == 0 + _, _, lse_idx_b = _dsa_fwd_flash_mla(q, kv, topk_idxs_aligned, 0.5, indexer_topk=0) + assert lse_idx_b is None, "(b) indexer_topk=0 must yield lse_indexer=None" + + # (c) 0 < indexer_topk < TopK + _, lse_c, lse_idx_c = _dsa_fwd_flash_mla( + q, kv, topk_idxs_aligned, 0.5, indexer_topk=TopK // 2 + ) + assert lse_idx_c is not None, "(c) lse_indexer should be present" + assert torch.equal( + lse_idx_c, stub.last_lse_indexer + ), "(c) lse_indexer should be kernel pass-through" + assert not torch.equal(lse_idx_c, lse_c), "(c) wrapper silently swapped lse_indexer for lse" + + # (d) indexer_topk == TopK -> fallback to lse.clone() + _, lse_d, lse_idx_d = _dsa_fwd_flash_mla(q, kv, topk_idxs_aligned, 0.5, indexer_topk=TopK) + assert torch.equal( + lse_idx_d, lse_d + ), "(d) lse_indexer should equal lse on TopK-cap fallback" + assert ( + lse_idx_d.data_ptr() != lse_d.data_ptr() + ), "(d) fallback should be a clone, not an alias" + + # ---- (e) topk_length + indexer_topk > 0 is rejected -------------- + # Use CPU tensors here — the assert fires before any kernel call. + with pytest.raises(AssertionError, match="indexer_topk > 0 requires non-compact"): + _dsa_fwd_flash_mla( + torch.zeros(2, 2, 512, dtype=torch.bfloat16), + torch.zeros(4, 512, dtype=torch.bfloat16), + torch.zeros(2, 8, dtype=torch.int32), + softmax_scale=0.5, + topk_length=torch.zeros(2, dtype=torch.int32), + indexer_topk=4, + ) + + +# --------------------------------------------------------------------------- +# indexer_topk — cudnn DSA wrapper for inference +# --------------------------------------------------------------------------- + + +class TestIndexerTopk: + """Indexer scoring + radix top-K wrapper. All three properties combined + in a single fixture; sub-block names appear in failure messages. + """ + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_indexer_topk_wrapper(self, reset_lazy_kernel_state): + """Combined assertions for: + + * (a) basic call: shapes / dtypes of the returned (topk_indices, + topk_length); kernels are called with the right BSHD layouts + and the SBHD-flat (b*sq, sk) scores; indexer_top_k kwargs. + * (b) topk > sk clamping: kernel call uses ``sk`` keys, trailing + ``[sk:]`` slots are -1, ``topk_length == sk``. + * (c) ``indexer_softmax_scale`` pre-scales the weights via the + ``relu(c·x) = c·relu(x)`` trick before reaching the kernel. + """ + + # ---- (a) basic call ---------------------------------------------- + sq, b, idx_nh, idx_hd = 6, 2, 4, 64 + sk = 12 + topk = 5 + ratio = 4 + + q_indexer = torch.randn(sq, b, idx_nh, idx_hd, dtype=torch.bfloat16, device='cuda') + k_indexer = torch.randn(sk, b, idx_hd, dtype=torch.bfloat16, device='cuda') + weights = torch.randn(sq, b, idx_nh, dtype=torch.bfloat16, device='cuda') + + scores = torch.randn(b, sq, sk, dtype=torch.float32, device='cuda') + captured = {} + + def fake_indexer_forward(q_bshd, k_bshd, w_bsh, ratio): + captured['indexer_forward'] = { + 'q_shape': q_bshd.shape, + 'k_shape': k_bshd.shape, + 'w_shape': w_bsh.shape, + 'ratio': ratio, + } + return {'scores': scores} + + def fake_filtered_topk(scores_flat, seq_lens, top_k, next_n, return_val): + captured['filtered_topk'] = { + 'scores_shape': scores_flat.shape, + 'seq_lens_shape': seq_lens.shape, + 'top_k': top_k, + 'next_n': next_n, + 'return_val': return_val, + } + n_rows = scores_flat.shape[0] + return {'indices': torch.zeros(n_rows, top_k, dtype=torch.int32, device='cuda')} + + fake_dsa = MagicMock() + fake_dsa.indexer_forward_wrapper.side_effect = fake_indexer_forward + fake_dsa.indexer_top_k_wrapper.side_effect = fake_filtered_topk + dk._DSA = fake_dsa + + topk_indices, topk_length = indexer_topk( + q_indexer, k_indexer, weights, topk=topk, ratio=ratio + ) + + assert topk_indices.shape == (b, sq, topk), "(a) topk_indices shape" + assert topk_indices.dtype == torch.int32, "(a) topk_indices dtype" + assert topk_length.shape == (b, sq), "(a) topk_length shape" + assert topk_length.dtype == torch.int32, "(a) topk_length dtype" + # BSHD / BSD layouts handed to the kernels. + assert captured['indexer_forward']['q_shape'] == ( + b, + sq, + idx_nh, + idx_hd, + ), "(a) indexer_forward q_shape" + assert captured['indexer_forward']['k_shape'] == ( + b, + sk, + 1, + idx_hd, + ), "(a) indexer_forward k_shape (must be unsqueezed h_kv=1)" + assert captured['indexer_forward']['w_shape'] == ( + b, + sq, + idx_nh, + ), "(a) indexer_forward w_shape" + assert captured['indexer_forward']['ratio'] == ratio, "(a) ratio kwarg" + assert captured['filtered_topk']['scores_shape'] == ( + b * sq, + sk, + ), "(a) topK scores_flat shape" + assert captured['filtered_topk']['seq_lens_shape'] == (b * sq,), "(a) topK seq_lens shape" + assert captured['filtered_topk']['top_k'] == min(topk, sk), "(a) top_k kwarg" + assert captured['filtered_topk']['next_n'] == 1, "(a) next_n kwarg" + assert captured['filtered_topk']['return_val'] is False, "(a) return_val kwarg" + + # ---- (b) topk > sk clamping -------------------------------------- + dk._DSA = None # force fresh mocks + sq2, b2, idx_nh2, idx_hd2 = 4, 1, 2, 32 + sk2 = 3 + topk2 = 8 # > sk + q2 = torch.randn(sq2, b2, idx_nh2, idx_hd2, dtype=torch.bfloat16, device='cuda') + k2 = torch.randn(sk2, b2, idx_hd2, dtype=torch.bfloat16, device='cuda') + w2 = torch.randn(sq2, b2, idx_nh2, dtype=torch.bfloat16, device='cuda') + scores2 = torch.zeros(b2, sq2, sk2, dtype=torch.float32, device='cuda') + kernel_indices2 = torch.zeros(b2 * sq2, sk2, dtype=torch.int32, device='cuda') + + fake_dsa_b = MagicMock() + fake_dsa_b.indexer_forward_wrapper.return_value = {'scores': scores2} + fake_dsa_b.indexer_top_k_wrapper.return_value = {'indices': kernel_indices2} + dk._DSA = fake_dsa_b + + topk_indices2, topk_length2 = indexer_topk(q2, k2, w2, topk=topk2, ratio=4) + assert topk_indices2.shape == (b2, sq2, topk2), "(b) padded topk_indices shape" + assert torch.all(topk_indices2[..., sk2:] == -1), "(b) trailing slots not -1" + assert torch.all(topk_length2 == sk2), "(b) topk_length should equal sk" + + # ---- (c) indexer_softmax_scale pre-scales weights --------------- + dk._DSA = None + sq3, b3, idx_nh3, idx_hd3 = 2, 1, 2, 32 + sk3 = 4 + scale = 0.125 + q3 = torch.zeros(sq3, b3, idx_nh3, idx_hd3, dtype=torch.bfloat16, device='cuda') + k3 = torch.zeros(sk3, b3, idx_hd3, dtype=torch.bfloat16, device='cuda') + w3 = torch.full((sq3, b3, idx_nh3), 8.0, dtype=torch.bfloat16, device='cuda') + captured_w = {} + + def fake_indexer_forward_c(q_bshd, k_bshd, w_bsh, ratio): + captured_w['w'] = w_bsh.detach().clone() + return {'scores': torch.zeros(b3, sq3, sk3, dtype=torch.float32, device='cuda')} + + fake_dsa_c = MagicMock() + fake_dsa_c.indexer_forward_wrapper.side_effect = fake_indexer_forward_c + fake_dsa_c.indexer_top_k_wrapper.return_value = { + 'indices': torch.zeros(b3 * sq3, sk3, dtype=torch.int32, device='cuda') + } + dk._DSA = fake_dsa_c + + indexer_topk(q3, k3, w3, topk=sk3, ratio=4, indexer_softmax_scale=scale) + expected_w = ( + (w3.float() * scale).to(torch.bfloat16).permute(1, 0, 2).reshape(b3, sq3, idx_nh3) + ) + assert torch.allclose( + captured_w['w'].float(), expected_w.float(), atol=1e-2, rtol=1e-2 + ), "(c) weights were not pre-scaled by indexer_softmax_scale" + + +# --------------------------------------------------------------------------- +# dsa_sparse_attn / SparseAttnFunc forward (mocked) +# --------------------------------------------------------------------------- + + +class TestDsaSparseAttn: + """Numerical fwd + bwd test for the public ``dsa_sparse_attn`` entry + point. The underlying kernels are mocked so the whole wrapper — including + the SBHD↔flat reshape on the forward and the autograd plumbing on the + backward — can be checked against deterministic ground truth. + """ + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_dsa_sparse_attn_fwd_and_bwd(self, reset_lazy_kernel_state): + """Combined forward + backward fixture (assertion blocks self-label + on failure): + + * (a) Forward output equals the FlashMLA stub's ``out`` reshaped + from flat ``(sq*b, np_, d_v)`` back to ``(sq, b, np_ * d_v)``. + * (b) Backward maps kernel grads onto the right SBHD leaf tensors + with the correct shapes; the bwd kernel is invoked exactly + once. + """ + sq, b, np_, d = 4, 2, 2, 512 + skv = 6 + TopK = _get_topk_alignment() + + query = torch.randn(sq, b, np_, d, dtype=torch.bfloat16, device='cuda', requires_grad=True) + kv = torch.randn(skv, b, d, dtype=torch.bfloat16, device='cuda', requires_grad=True) + attn_sink = torch.zeros(np_, dtype=torch.float32, device='cuda', requires_grad=True) + topk_idxs = torch.zeros(sq * b, TopK, dtype=torch.int32, device='cuda') + + # Coordinated stubs: FlashMLA fwd + cuDNN sparse-attn bwd, both + # deterministic so every gradient slot is independently verifiable. + flash_stub = _make_flash_mla_stub(d_v=d) + dq_kernel = torch.full((sq * b, np_, d), 7.0, dtype=torch.bfloat16, device='cuda') + dkv_kernel = torch.full((skv * b, d), -3.0, dtype=torch.bfloat16, device='cuda') + d_sink_kernel = torch.full((np_,), 11.0, dtype=torch.float32, device='cuda') + fake_dsa = MagicMock() + fake_dsa.sparse_attention_backward_wrapper.return_value = { + 'dq': dq_kernel, + 'dkv': dkv_kernel, + 'd_sink': d_sink_kernel, + } + dk._flash_mla_sparse_fwd = flash_stub + dk._DSA = fake_dsa + + out = dsa_sparse_attn(query, kv, attn_sink, topk_idxs, softmax_scale=0.5) + + # ---- (a) forward ------------------------------------------------ + assert out.shape == (sq, b, np_ * d), "(a) forward shape" + assert out.dtype == torch.bfloat16, "(a) forward dtype" + expected_out = flash_stub.last_out.reshape(sq, b, np_, d).reshape(sq, b, np_ * d) + assert torch.equal(out, expected_out), "(a) forward value pass-through" + + # ---- (b) backward ----------------------------------------------- + out.sum().backward() + assert query.grad is not None, "(b) query.grad missing" + assert kv.grad is not None, "(b) kv.grad missing" + assert attn_sink.grad is not None, "(b) attn_sink.grad missing" + assert torch.equal( + query.grad, dq_kernel.reshape(sq, b, np_, d) + ), "(b) query.grad mis-reshaped" + assert torch.equal(kv.grad, dkv_kernel.reshape(skv, b, d)), "(b) kv.grad mis-reshaped" + assert torch.equal(attn_sink.grad, d_sink_kernel), "(b) attn_sink.grad mismatch" + fake_dsa.sparse_attention_backward_wrapper.assert_called_once() + + +# --------------------------------------------------------------------------- +# fused_indexer_sparse_attn — Path B autograd Function (mocked) +# --------------------------------------------------------------------------- + + +def _install_full_dsa_mock( + *, + b: int, + sq: int, + np_: int, + d: int, + n_comp: int, + idx_nh: int, + predict_fn=None, + target_fn=None, + dq_value: float = 7.0, + dkv_value: float = -3.0, + d_sink_value: float = 11.0, + d_index_q_value: float = 0.5, + d_weights_value: float = -0.25, + d_index_k_value: float = 1.5, +): + """Patch the module-level ``_DSA`` and ``_flash_mla_sparse_fwd`` slots + with a coordinated set of deterministic stubs covering every kernel + invoked by :class:`FusedIndexerSparseAttnFunc`. + + ``predict_fn`` / ``target_fn`` (if provided) build the per-row + distribution given ``(b, sq, topk, device)``. By default both return a + uniform ``1/topk`` distribution, which yields ``KL(target || predict) = 0`` + so the loss is exactly zero. + + All backward kernels return constant-filled tensors so each gradient slot + can be independently verified. + """ + + if predict_fn is None: + predict_fn = lambda B, S, K, dev: torch.full( + (B, S, K), 1.0 / max(K, 1), dtype=torch.float32, device=dev + ) + if target_fn is None: + target_fn = predict_fn + + fake_dsa = MagicMock(name='_DSA_full_stub') + + def fake_indexer_forward(q_bshd, k_bshd, w_bsh, ratio): + return {'scores': torch.zeros(b, sq, n_comp, dtype=torch.float32, device=q_bshd.device)} + + fake_dsa.indexer_forward_wrapper.side_effect = fake_indexer_forward + + def fake_filtered_topk(scores_flat, seq_lens, top_k, next_n, return_val): + return { + 'indices': torch.zeros( + scores_flat.shape[0], top_k, dtype=torch.int32, device=scores_flat.device + ) + } + + fake_dsa.indexer_top_k_wrapper.side_effect = fake_filtered_topk + + def fake_sparse_indexer_score_backward(q, k, w, topk_indices, qhead_per_kv_head): + topk = topk_indices.shape[-1] + return {'predict': predict_fn(b, sq, topk, q.device)} + + fake_dsa.sparse_indexer_score_recompute_wrapper.side_effect = fake_sparse_indexer_score_backward + + def fake_sparse_attn_score_backward(q, k, lse, topk_indices, sm_scale, qhead_per_kv_head): + topk = topk_indices.shape[-1] + return {'target': target_fn(b, sq, topk, q.device)} + + fake_dsa.sparse_attn_score_recompute_wrapper.side_effect = fake_sparse_attn_score_backward + + def fake_sparse_attn_backward(q, kv, out, dout, lse, attn_sink, topk_idxs, **kwargs): + return { + 'dq': torch.full_like(q, dq_value), + 'dkv': torch.full_like(kv, dkv_value), + 'd_sink': torch.full_like(attn_sink, d_sink_value), + } + + fake_dsa.sparse_attention_backward_wrapper.side_effect = fake_sparse_attn_backward + + def fake_indexer_grad_backward( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score, + index_score, + topk_indices, + sm_scale, + loss_coeff, + grad_loss, + block_I, + ): + return { + 'd_index_q': torch.full_like(q_idx_bshd, d_index_q_value), + 'd_weights': torch.full_like(w_bsh, d_weights_value), + 'd_index_k': torch.full_like(k_idx_bsd, d_index_k_value), + } + + fake_dsa.indexer_backward_wrapper.side_effect = fake_indexer_grad_backward + + flash_stub = _make_flash_mla_stub(d_v=d) + + dk._DSA = fake_dsa + dk._flash_mla_sparse_fwd = flash_stub + return fake_dsa, flash_stub + + +def _install_full_dsa_mock_dense( + *, + b: int, + sq: int, + np_: int, + d: int, + n_comp: int, + idx_nh: int, + target_score_fn=None, + target_l1norm_fn=None, + predict_score_fn=None, + predict_lse_fn=None, + dq_value: float = 7.0, + dkv_value: float = -3.0, + d_sink_value: float = 11.0, + d_index_q_value: float = 0.5, + d_weights_value: float = -0.25, + d_index_k_value: float = 1.5, +): + """Coordinated stubs covering the dense-loss (``sparse_loss=False``) path. + + Mirrors :func:`_install_full_dsa_mock` for the sparse path, but stubs + the four dense-only kernel wrappers: + + * ``dense_indexer_score_recompute_wrapper`` -> ``(out, denom=index_lse)`` + * ``dense_attn_score_recompute_wrapper`` -> ``(out, denom=attn_l1norm)`` + * ``dense_indexer_backward_wrapper`` -> ``{d_index_q, d_weights, d_index_k}`` + + Defaults make ``target == predict == uniform(1/n_comp)`` so KL == 0. + Override the four ``*_fn`` callables to drive the loss to known + analytical values; each callable receives ``(B, S_q, S_k, device)`` and + returns the score tensor (``S_k``-dim) or denom (no ``S_k`` dim). + """ + + if target_score_fn is None: + target_score_fn = lambda B, S, K, dev: torch.full( + (B, S, K), 1.0 / max(K, 1), dtype=torch.float32, device=dev + ) + if target_l1norm_fn is None: + target_l1norm_fn = lambda B, S, K, dev: torch.ones((B, S), dtype=torch.float32, device=dev) + if predict_score_fn is None: + predict_score_fn = lambda B, S, K, dev: torch.zeros( + (B, S, K), dtype=torch.float32, device=dev + ) + if predict_lse_fn is None: + predict_lse_fn = lambda B, S, K, dev: torch.full( + (B, S), float(math.log(max(K, 1))), dtype=torch.float32, device=dev + ) + + fake_dsa = MagicMock(name='_DSA_full_dense_stub') + + def fake_indexer_forward(q_bshd, k_bshd, w_bsh, ratio): + return {'scores': torch.zeros(b, sq, n_comp, dtype=torch.float32, device=q_bshd.device)} + + fake_dsa.indexer_forward_wrapper.side_effect = fake_indexer_forward + + def fake_filtered_topk(scores_flat, seq_lens, top_k, next_n, return_val): + return { + 'indices': torch.zeros( + scores_flat.shape[0], top_k, dtype=torch.int32, device=scores_flat.device + ) + } + + fake_dsa.indexer_top_k_wrapper.side_effect = fake_filtered_topk + + def fake_dense_indexer_score(q, k, w, qhead_per_kv_head, sm_scale, ratio): + dev = q.device + return { + 'out': predict_score_fn(b, sq, n_comp, dev), + 'denom': predict_lse_fn(b, sq, n_comp, dev), + } + + fake_dsa.dense_indexer_score_recompute_wrapper.side_effect = fake_dense_indexer_score + + def fake_dense_attn_score(q, k, lse, softmax_scale, qhead_per_kv_head, ratio): + dev = q.device + return { + 'out': target_score_fn(b, sq, n_comp, dev), + 'denom': target_l1norm_fn(b, sq, n_comp, dev), + } + + fake_dsa.dense_attn_score_recompute_wrapper.side_effect = fake_dense_attn_score + + def fake_sparse_attn_backward(q, kv, out, dout, lse, attn_sink, topk_idxs, **kwargs): + return { + 'dq': torch.full_like(q, dq_value), + 'dkv': torch.full_like(kv, dkv_value), + 'd_sink': torch.full_like(attn_sink, d_sink_value), + } + + fake_dsa.sparse_attention_backward_wrapper.side_effect = fake_sparse_attn_backward + + def fake_dense_indexer_grad_backward( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score, + attn_l1norm, + index_score, + index_lse, + sm_scale, + loss_coeff, + grad_loss, + ratio, + block_I, + ): + return { + 'd_index_q': torch.full_like(q_idx_bshd, d_index_q_value), + 'd_weights': torch.full_like(w_bsh, d_weights_value), + 'd_index_k': torch.full_like(k_idx_bsd, d_index_k_value), + } + + fake_dsa.dense_indexer_backward_wrapper.side_effect = fake_dense_indexer_grad_backward + + flash_stub = _make_flash_mla_stub(d_v=d) + + dk._DSA = fake_dsa + dk._flash_mla_sparse_fwd = flash_stub + return fake_dsa, flash_stub + + +class TestFusedIndexerSparseAttn: + """End-to-end numerical tests for the Path B autograd Function with all + underlying CUDA kernels mocked. + """ + + # Common shapes shared across the forward tests. + SHAPES = dict(sq=4, b=2, np_=2, d=512, skv=8, n_comp=4, idx_nh=4, idx_hd=64) + + def _make_inputs(self, *, requires_grad=False): + """Build the seven differentiable + one non-differentiable inputs.""" + s = self.SHAPES + win_topk = _get_topk_alignment() - 2 # exercise padding + torch.manual_seed(0) + + def make(*shape, dtype, rg=False): + t = torch.randn(*shape, dtype=dtype, device='cuda') + if requires_grad and rg: + t = t.detach().clone().requires_grad_(True) + return t + + query = make(s['sq'], s['b'], s['np_'], s['d'], dtype=torch.bfloat16, rg=True) + kv_full = make(s['skv'], s['b'], s['d'], dtype=torch.bfloat16, rg=True) + attn_sink = torch.zeros(s['np_'], dtype=torch.float32, device='cuda') + if requires_grad: + attn_sink = attn_sink.detach().clone().requires_grad_(True) + window_idxs = torch.zeros(s['b'], s['sq'], win_topk, dtype=torch.int32, device='cuda') + q_indexer = make(s['sq'], s['b'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, rg=True) + k_indexer = make(s['n_comp'], s['b'], s['idx_hd'], dtype=torch.bfloat16, rg=True) + weights = make(s['sq'], s['b'], s['idx_nh'], dtype=torch.bfloat16, rg=True) + return dict( + query=query, + kv_full=kv_full, + attn_sink=attn_sink, + window_idxs=window_idxs, + q_indexer=q_indexer, + k_indexer=k_indexer, + weights=weights, + ) + + @pytest.mark.parametrize( + "loss_coeff, target_kind, expected", + [ + # KL(target == predict) == 0 → loss == 0 regardless of coeff. + (1.0, 'uniform', 0.0), + # loss_coeff == 0 short-circuits even when target != predict. + (0.0, 'peaked', 0.0), + # target = δ_0, predict = uniform(1/K) → KL = log(K) per row, + # mean over rows = log(K), scaled by coeff = coeff * log(K). + (0.7, 'peaked', 0.7 * math.log(2)), + # Linearity in loss_coeff: doubling the coeff doubles the loss. + (2.0, 'peaked', 2.0 * math.log(2)), + ], + ids=['identical_dists', 'coeff_zero', 'analytical_kl', 'linearity_x2'], + ) + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_indexer_loss_formula(self, loss_coeff, target_kind, expected, reset_lazy_kernel_state): + """All four loss-property cases share one fixture: + + * KL is zero when target == predict, + * ``loss_coeff == 0`` short-circuits to zero, + * for ``target = δ_0`` and ``predict = uniform(1/K)`` the per-row + KL is exactly ``log(K)`` so the mean is ``loss_coeff * log(K)``, + * the loss is linear in ``loss_coeff``. + """ + s = self.SHAPES + topk = 2 # = effective_topk = min(indexer_topk, n_comp); appears as K + target_fn = ( + _uniform_dist + if target_kind == 'uniform' + else (lambda B, S, K, dev: _peaked_dist(B, S, K, dev, peak_idx=0)) + ) + + inputs = self._make_inputs() + _install_full_dsa_mock( + b=s['b'], + sq=s['sq'], + np_=s['np_'], + d=s['d'], + n_comp=s['n_comp'], + idx_nh=s['idx_nh'], + predict_fn=_uniform_dist, + target_fn=target_fn, + ) + + _, indexer_loss = fused_indexer_sparse_attn( + **inputs, + indexer_topk=topk, + ratio=4, + softmax_scale=0.5, + loss_coeff=loss_coeff, + sparse_loss=True, + kv_offset=s['skv'] - s['n_comp'], + ) + + assert torch.allclose( + indexer_loss, torch.tensor(expected, device='cuda'), rtol=1e-5, atol=1e-5 + ), f"got {indexer_loss.item()}, expected {expected}" + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_sparse_path_fwd_output_bwd_grads_and_topk_clamp(self, reset_lazy_kernel_state): + """Combined coverage for the sparse-loss path's three non-numerical + properties (assertion blocks self-label on failure): + + * (a) ``output`` is exactly the FlashMLA stub's ``out`` reshaped + from ``(sq*b, np_, d_v)`` to ``(sq, b, np_ * d_v)``. + * (b) After backward, each leaf gradient equals the corresponding + mocked kernel output, with q/kv/attn_sink coming from the + sparse-attn bwd kernel and q_indexer/k_indexer/weights coming + from the indexer bwd kernel (BSHD → SBHD permute applied). + * (c) ``indexer_topk > n_comp`` is clamped to ``n_comp`` before the + radix TopK kernel is called. + """ + s = self.SHAPES + + # ---- (a) forward pass-through (no grads needed) ------------------ + inputs = self._make_inputs() + _, flash_stub_a = _install_full_dsa_mock( + b=s['b'], sq=s['sq'], np_=s['np_'], d=s['d'], n_comp=s['n_comp'], idx_nh=s['idx_nh'] + ) + output_a, _ = fused_indexer_sparse_attn( + **inputs, + indexer_topk=2, + ratio=4, + softmax_scale=0.5, + indexer_softmax_scale=0.125, + loss_coeff=0.0, + sparse_loss=True, + kv_offset=s['skv'] - s['n_comp'], + ) + assert output_a.shape == (s['sq'], s['b'], s['np_'] * s['d']), "(a) shape" + assert output_a.dtype == torch.bfloat16, "(a) dtype" + expected_a = flash_stub_a.last_out.reshape(s['sq'], s['b'], s['np_'], s['d']).reshape( + s['sq'], s['b'], s['np_'] * s['d'] + ) + assert torch.equal(output_a, expected_a), "(a) forward value pass-through" + + # ---- (b) backward grad propagation ------------------------------- + dk._DSA = None # fresh mocks + dk._flash_mla_sparse_fwd = None + inputs_b = self._make_inputs(requires_grad=True) + _install_full_dsa_mock( + b=s['b'], + sq=s['sq'], + np_=s['np_'], + d=s['d'], + n_comp=s['n_comp'], + idx_nh=s['idx_nh'], + dq_value=7.0, + dkv_value=-3.0, + d_sink_value=11.0, + d_index_q_value=0.5, + d_weights_value=-0.25, + d_index_k_value=1.5, + ) + output_b, indexer_loss_b = fused_indexer_sparse_attn( + **inputs_b, + indexer_topk=2, + ratio=4, + softmax_scale=0.5, + indexer_softmax_scale=0.125, + loss_coeff=1.0, + sparse_loss=True, + kv_offset=s['skv'] - s['n_comp'], + ) + (output_b.sum() + indexer_loss_b).backward() + for name, value in [ + ('query', 7.0), + ('kv_full', -3.0), + ('attn_sink', 11.0), + ('q_indexer', 0.5), + ('k_indexer', 1.5), + ('weights', -0.25), + ]: + grad = inputs_b[name].grad + assert grad is not None, f"(b) {name}: missing grad" + assert torch.equal(grad, torch.full_like(inputs_b[name], value)), ( + f"(b) {name}: grad does not equal full({value}); " + f"got first elem = {grad.float().flatten()[0].item()}" + ) + + # ---- (c) indexer_topk > n_comp clamp ----------------------------- + dk._DSA = None + dk._flash_mla_sparse_fwd = None + inputs_c = self._make_inputs() + fake_dsa_c, _ = _install_full_dsa_mock( + b=s['b'], sq=s['sq'], np_=s['np_'], d=s['d'], n_comp=s['n_comp'], idx_nh=s['idx_nh'] + ) + fused_indexer_sparse_attn( + **inputs_c, + indexer_topk=999, # > n_comp + ratio=4, + softmax_scale=0.5, + loss_coeff=0.0, + sparse_loss=True, + kv_offset=s['skv'] - s['n_comp'], + ) + topk_call = fake_dsa_c.indexer_top_k_wrapper.call_args + assert ( + topk_call.kwargs['top_k'] == s['n_comp'] + ), f"(c) top_k clamp: got {topk_call.kwargs['top_k']}, expected {s['n_comp']}" + + +# --------------------------------------------------------------------------- +# fused_indexer_sparse_attn — dense path (sparse_loss=False) +# --------------------------------------------------------------------------- + + +class TestDenseFusedIndexerSparseAttn: + """End-to-end tests for the dense-loss branch of Path B with all + underlying CUDA kernels mocked. Mirrors :class:`TestFusedIndexerSparseAttn` + but exercises the ``sparse_loss=False`` code path through + :class:`FusedIndexerSparseAttnFunc`. + """ + + SHAPES = dict(sq=4, b=2, np_=2, d=512, skv=8, n_comp=4, idx_nh=4, idx_hd=64) + + def _make_inputs(self, *, requires_grad=False): + s = self.SHAPES + win_topk = _get_topk_alignment() - 2 # exercise padding + torch.manual_seed(0) + + def make(*shape, dtype, rg=False): + t = torch.randn(*shape, dtype=dtype, device='cuda') + if requires_grad and rg: + t = t.detach().clone().requires_grad_(True) + return t + + query = make(s['sq'], s['b'], s['np_'], s['d'], dtype=torch.bfloat16, rg=True) + kv_full = make(s['skv'], s['b'], s['d'], dtype=torch.bfloat16, rg=True) + attn_sink = torch.zeros(s['np_'], dtype=torch.float32, device='cuda') + if requires_grad: + attn_sink = attn_sink.detach().clone().requires_grad_(True) + window_idxs = torch.zeros(s['b'], s['sq'], win_topk, dtype=torch.int32, device='cuda') + q_indexer = make(s['sq'], s['b'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, rg=True) + k_indexer = make(s['n_comp'], s['b'], s['idx_hd'], dtype=torch.bfloat16, rg=True) + weights = make(s['sq'], s['b'], s['idx_nh'], dtype=torch.bfloat16, rg=True) + return dict( + query=query, + kv_full=kv_full, + attn_sink=attn_sink, + window_idxs=window_idxs, + q_indexer=q_indexer, + k_indexer=k_indexer, + weights=weights, + ) + + @pytest.mark.parametrize( + "loss_coeff, target_kind, expected", + [ + # Identical dists: KL == 0 regardless of coeff. + (1.0, 'uniform', 0.0), + # loss_coeff == 0 short-circuits even when target != predict. + (0.0, 'peaked', 0.0), + # target = δ_0, predict = uniform(1/n_comp) + # per-row KL = log(n_comp); mean = log(n_comp); loss = coeff * log(n_comp). + # n_comp = 4 here. + (0.7, 'peaked', 0.7 * math.log(4)), + (2.0, 'peaked', 2.0 * math.log(4)), + ], + ids=['identical_dists', 'coeff_zero', 'analytical_kl', 'linearity_x2'], + ) + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_dense_indexer_loss_formula( + self, loss_coeff, target_kind, expected, reset_lazy_kernel_state + ): + """``_kl_loss_from_dense_scores`` is the dense analogue of + ``_kl_loss_from_target_predict``. Verifies the same four KL + properties (zero, coeff-zero short-circuit, analytical formula, + linearity in coeff) over the dense ``(B, S_q, S_k)`` tensors. + + We drive the stub outputs so that: + + * predict = ``softmax(0)`` over S_k = uniform(1/n_comp). This is + encoded as ``index_score = 0`` everywhere, ``index_lse = log(n_comp)``; + ``predict = exp(score - lse) = 1/n_comp``. + * For ``target = uniform``: attn_score = 1/n_comp uniformly, attn_l1norm = 1. + * For ``target = δ_0``: attn_score peaked on slot 0 with sum 1, attn_l1norm = 1. + """ + s = self.SHAPES + + if target_kind == 'uniform': + target_score_fn = lambda B, S, K, dev: torch.full( + (B, S, K), 1.0 / max(K, 1), dtype=torch.float32, device=dev + ) + else: + + def target_score_fn(B, S, K, dev): + t = torch.zeros((B, S, K), dtype=torch.float32, device=dev) + t[..., 0] = 1.0 + return t + + target_l1norm_fn = lambda B, S, K, dev: torch.ones((B, S), dtype=torch.float32, device=dev) + + inputs = self._make_inputs() + _install_full_dsa_mock_dense( + b=s['b'], + sq=s['sq'], + np_=s['np_'], + d=s['d'], + n_comp=s['n_comp'], + idx_nh=s['idx_nh'], + target_score_fn=target_score_fn, + target_l1norm_fn=target_l1norm_fn, + # predict_score_fn / predict_lse_fn defaults give uniform predict. + ) + + _, indexer_loss = fused_indexer_sparse_attn( + **inputs, + indexer_topk=2, + ratio=4, + softmax_scale=0.5, + loss_coeff=loss_coeff, + sparse_loss=False, + kv_offset=s['skv'] - s['n_comp'], + ) + + assert torch.allclose( + indexer_loss, torch.tensor(expected, device='cuda'), rtol=1e-5, atol=1e-5 + ), f"got {indexer_loss.item()}, expected {expected}" + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_dense_path_fwd_kernel_calls_and_bwd_grads(self, reset_lazy_kernel_state): + """Combined coverage for the dense-loss path's two non-numerical + properties (assertion blocks self-label on failure): + + * (a) The forward invokes ``dense_attn_score_recompute_wrapper`` + (NOT the sparse score kernels) with the right BSHD/4-D shapes + and ``ratio`` / scale args. The indexer predict is derived + directly from ``indexer_forward_wrapper`` scores (no separate + ``dense_indexer_score_recompute_wrapper`` call). + * (b) The forward eagerly invokes ``dense_indexer_backward_wrapper`` + (NOT the sparse one), threads ``ratio`` through, and the + resulting grads land on the right SBHD leaves (BSHD → SBHD + permute applied for the indexer-side grads, scaled by + ``grad_loss`` in the actual backward). + """ + s = self.SHAPES + ratio = 4 + softmax_scale = 0.5 + idx_scale = 0.125 + loss_coeff = 1.0 + + # ---- (a) forward kernel selection + arg shapes ------------------- + inputs_a = self._make_inputs() + fake_dsa_a, _ = _install_full_dsa_mock_dense( + b=s['b'], sq=s['sq'], np_=s['np_'], d=s['d'], n_comp=s['n_comp'], idx_nh=s['idx_nh'] + ) + fused_indexer_sparse_attn( + **inputs_a, + indexer_topk=2, + ratio=ratio, + softmax_scale=softmax_scale, + indexer_softmax_scale=idx_scale, + loss_coeff=loss_coeff, + sparse_loss=False, + kv_offset=s['skv'] - s['n_comp'], + ) + # Indexer predict is derived from indexer_forward_wrapper scores + # (gather + logsumexp), NOT from dense_indexer_score_recompute_wrapper. + fake_dsa_a.dense_indexer_score_recompute_wrapper.assert_not_called() + fake_dsa_a.dense_attn_score_recompute_wrapper.assert_called_once() + fake_dsa_a.sparse_indexer_score_recompute_wrapper.assert_not_called() + fake_dsa_a.sparse_attn_score_recompute_wrapper.assert_not_called() + + attn_call = fake_dsa_a.dense_attn_score_recompute_wrapper.call_args + q_attn, k_attn, lse_arg, sm_arg = attn_call.args + assert q_attn.shape == (s['b'], s['sq'], s['np_'], s['d']), "(a) dense attn score: q shape" + assert k_attn.shape == ( + s['b'], + s['n_comp'], + 1, + s['d'], + ), "(a) dense attn score: k shape (h_kv=1)" + assert lse_arg.shape == (s['b'], s['sq'], s['np_']), "(a) dense attn score: lse shape" + assert sm_arg == softmax_scale, "(a) dense attn score: positional softmax_scale" + assert attn_call.kwargs['qhead_per_kv_head'] == s['np_'] + assert attn_call.kwargs['ratio'] == ratio + + # ---- (b) forward-eager indexer backward + grad propagation -------- + dk._DSA = None + dk._flash_mla_sparse_fwd = None + inputs_b = self._make_inputs(requires_grad=True) + fake_dsa_b, _ = _install_full_dsa_mock_dense( + b=s['b'], + sq=s['sq'], + np_=s['np_'], + d=s['d'], + n_comp=s['n_comp'], + idx_nh=s['idx_nh'], + dq_value=7.0, + dkv_value=-3.0, + d_sink_value=11.0, + d_index_q_value=0.5, + d_weights_value=-0.25, + d_index_k_value=1.5, + ) + output, indexer_loss = fused_indexer_sparse_attn( + **inputs_b, + indexer_topk=2, + ratio=ratio, + softmax_scale=softmax_scale, + indexer_softmax_scale=idx_scale, + loss_coeff=loss_coeff, + sparse_loss=False, + kv_offset=s['skv'] - s['n_comp'], + ) + (output.sum() + indexer_loss).backward() + + # dense_indexer_backward_wrapper is called eagerly during forward. + fake_dsa_b.dense_indexer_backward_wrapper.assert_called_once() + fake_dsa_b.indexer_backward_wrapper.assert_not_called() + ig_call = fake_dsa_b.dense_indexer_backward_wrapper.call_args + assert ig_call.kwargs['ratio'] == ratio, "(b) ratio not threaded through" + assert ig_call.kwargs['sm_scale'] == idx_scale, "(b) sm_scale not threaded" + assert ig_call.kwargs['loss_coeff'] == loss_coeff, "(b) loss_coeff not threaded" + + for name, value in [ + ('query', 7.0), + ('kv_full', -3.0), + ('attn_sink', 11.0), + ('q_indexer', 0.5), + ('k_indexer', 1.5), + ('weights', -0.25), + ]: + grad = inputs_b[name].grad + assert grad is not None, f"(b) {name}: missing grad" + assert torch.equal( + grad, torch.full_like(inputs_b[name], value) + ), f"(b) {name}: grad does not equal full({value})" + + +# --------------------------------------------------------------------------- +# Real-kernel parity tests (cuDNN + optional FlashMLA) +# --------------------------------------------------------------------------- +# +# Everything above this banner stubs ``cudnn.DSA`` and ``flash_mla`` with +# ``MagicMock``-based fakes; that exercises the Python plumbing of +# ``dsa_kernels.py`` (shape transforms, autograd wiring, KL composition) +# but does NOT verify that the cuDNN kernels themselves compute what +# ``dsa_kernels.py`` expects them to compute. +# +# The tests below close that gap by running each helper / public function +# end-to-end against a small PyTorch reference implementation. Numeric +# tolerances are bf16-friendly (atol/rtol ~ 5e-2 for raw scores, 1e-3 for +# normalized distributions, 5e-2 for backward grads). +# +# Skipped automatically when: +# * CUDA is unavailable; +# * cuDNN frontend is not installed (``import cudnn`` fails); +# * ``cudnn.DSA`` namespace is missing; +# * SM is too low (sparse: SM90+; dense: SM100+); +# * for FlashMLA-needing tests, ``flash_mla`` is not installed. +# --------------------------------------------------------------------------- + + +def _skip_if_real_kernels_unavailable(*, sm_min: int = 9, need_flash_mla: bool = False): + """Pytest-side gate for real-kernel tests. Raises ``pytest.skip`` if + any of the runtime dependencies are missing. + """ + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + sm_major = torch.cuda.get_device_capability()[0] + if sm_major < sm_min: + pytest.skip(f"requires SM{sm_min}+, found SM{sm_major}") + cudnn = pytest.importorskip("cudnn") + from packaging.version import Version + + if Version(cudnn.__version__) < Version("1.24.0"): + pytest.skip(f"requires cudnn>=1.24.0, found {cudnn.__version__}") + if not hasattr(cudnn, 'DSA'): + pytest.skip("cudnn.DSA namespace not available") + if need_flash_mla: + pytest.importorskip("flash_mla") + + +# --------------------------------------------------------------------------- +# PyTorch reference implementations +# --------------------------------------------------------------------------- + + +def _ratio_causal_valid_mask(sq: int, sk: int, ratio: int, device) -> torch.Tensor: + """``(Sq, Sk)`` bool: valid iff ``k_idx < min(Sk, (q_idx + 1) // ratio)``. + + Matches the cuDNN dense-score kernels' built-in causal mask + (``col_limit = min(S_k, (q + 1) // ratio)``) and ``csa.py``'s + ``compress_ratio`` mask formulation. + """ + q_idx = torch.arange(sq, device=device).unsqueeze(1) # (Sq, 1) + k_idx = torch.arange(sk, device=device).unsqueeze(0) # (1, Sk) + col_limit = ((q_idx + 1) // ratio).clamp(max=sk) + return k_idx < col_limit # (Sq, Sk) + + +def _ref_indexer_full_score( + q_bshd_fp32: torch.Tensor, # (B, Sq, H, D) + k_bsd_fp32: torch.Tensor, # (B, Sk, D) — MQA + w_bsh_fp32: torch.Tensor, # (B, Sq, H) + sm_scale: float, + ratio: int, +) -> torch.Tensor: + """Reference for ``_bwd_dense_indexer_score.out``. + + ``S[b,q,k] = sm_scale * sum_h ReLU(Q[b,q,h] @ K[b,k]^T) * W[b,q,h]``, + with the kernel's bottom-right ratio causal mask producing ``-inf`` + at masked positions. + """ + B, Sq, _, _ = q_bshd_fp32.shape + Sk = k_bsd_fp32.shape[1] + qk = torch.einsum('bqhd,bkd->bqhk', q_bshd_fp32, k_bsd_fp32) # (B, Sq, H, Sk) + relu_qk = torch.relu(qk) + s = (relu_qk * w_bsh_fp32.unsqueeze(-1)).sum(dim=2) * sm_scale # (B, Sq, Sk) + valid = _ratio_causal_valid_mask(Sq, Sk, ratio, s.device).unsqueeze(0) + return torch.where(valid, s, torch.full_like(s, float('-inf'))) + + +def _ref_attn_full_score( + q_bshd_fp32: torch.Tensor, # (B, Sq, H, D) + k_bsd_fp32: torch.Tensor, # (B, Sk, D) — MQA + lse_bshq_fp32: torch.Tensor, # (B, Sq, H) + softmax_scale: float, + ratio: int, +) -> torch.Tensor: + """Reference for ``_bwd_dense_attn_score.out``. + + ``out[b,q,k] = sum_h exp(Q[b,q,h] @ K[b,k]^T * scale - LSE[b,q,h])``, + with the ratio causal mask producing ``0`` at masked positions + (the per-head ``exp`` is zeroed out, contributing nothing to the sum). + """ + B, Sq, _, _ = q_bshd_fp32.shape + Sk = k_bsd_fp32.shape[1] + qk = torch.einsum('bqhd,bkd->bqhk', q_bshd_fp32, k_bsd_fp32) * softmax_scale + p = torch.exp(qk - lse_bshq_fp32.unsqueeze(-1)) + s = p.sum(dim=2) # (B, Sq, Sk) + valid = _ratio_causal_valid_mask(Sq, Sk, ratio, s.device).unsqueeze(0) + return torch.where(valid, s, torch.zeros_like(s)) + + +def _ref_indexer_predict_sparse(q_bshd_fp32, k_bsd_fp32, w_bsh_fp32, topk_indices, sm_scale): + """Reference for ``sparse_indexer_score_recompute_wrapper.predict``. + + Compute the full-KV indexer score, gather ``topk_indices``, softmax + over the topK axis. ``-1`` entries in topk are masked to ``-inf`` + so they contribute zero probability. + """ + qk = torch.einsum('bqhd,bkd->bqhk', q_bshd_fp32, k_bsd_fp32) + s = (torch.relu(qk) * w_bsh_fp32.unsqueeze(-1)).sum(dim=2) * sm_scale # (B, Sq, Sk) + valid = topk_indices >= 0 + safe = topk_indices.clamp(min=0).long() + s_topk = torch.gather(s, dim=-1, index=safe) + s_topk = torch.where(valid, s_topk, torch.full_like(s_topk, float('-inf'))) + return torch.softmax(s_topk, dim=-1) + + +def _ref_attn_target_sparse(q_bshd_fp32, k_bsd_fp32, lse_bsh_fp32, topk_indices, softmax_scale): + """Reference for ``sparse_attn_score_recompute_wrapper.target``. + + Per-head ``exp(QK*scale - LSE)``, sum over heads, gather topK, + L1-normalise over the topK axis. ``-1`` entries are zero-masked + pre-normalisation. + """ + qk = torch.einsum('bqhd,bkd->bqhk', q_bshd_fp32, k_bsd_fp32) * softmax_scale + p = torch.exp(qk - lse_bsh_fp32.unsqueeze(-1)) # (B, Sq, H, Sk) + s = p.sum(dim=2) # (B, Sq, Sk) + valid = topk_indices >= 0 + safe = topk_indices.clamp(min=0).long() + s_topk = torch.gather(s, dim=-1, index=safe) + s_topk = torch.where(valid, s_topk, torch.zeros_like(s_topk)) + denom = s_topk.sum(dim=-1, keepdim=True).clamp(min=1e-10) + return s_topk / denom + + +def _ref_dense_indexer_loss( + q_indexer_bshd_fp32, + k_indexer_bsd_fp32, + w_bsh_fp32, + q_attn_bshd_fp32, + k_attn_bsd_fp32, + lse_bshq_fp32, + indexer_softmax_scale: float, + attn_softmax_scale: float, + ratio: int, + loss_coeff: float, +) -> torch.Tensor: + """Reference dense KL loss (matches ``compute_dsa_indexer_loss(sparse_loss=False)`` + in ``dsa.py``). Uses the same ratio causal mask the kernel applies. + """ + eps = torch.finfo(torch.float32).tiny + # Per-(b,q,k) raw scores via the same formulas the kernels use. + attn_scores = _ref_attn_full_score( + q_attn_bshd_fp32, k_attn_bsd_fp32, lse_bshq_fp32, attn_softmax_scale, ratio + ) # (B, Sq, Sk) head-summed, ratio-masked, zeros at masked positions. + index_scores = _ref_indexer_full_score( + q_indexer_bshd_fp32, k_indexer_bsd_fp32, w_bsh_fp32, indexer_softmax_scale, ratio + ) # (B, Sq, Sk) ReLU·W, ratio-masked, -inf at masked positions. + + # L1-norm denom for target; LSE for predict. + attn_denom = attn_scores.sum(dim=-1) # (B, Sq) + index_lse = torch.logsumexp(index_scores, dim=-1) # (B, Sq), -inf for fully-masked rows + + row_valid = (attn_denom > eps) & torch.isfinite(index_lse) + + safe_l1 = attn_denom.clamp(min=eps) + safe_lse = torch.where(row_valid, index_lse, torch.zeros_like(index_lse)) + + target = attn_scores / safe_l1.unsqueeze(-1) + target_clamped = target.clamp(min=eps) + # Mask within-row: ratio-causal-masked positions have ``index_scores = + # -inf`` (from ``_ref_indexer_full_score``). Letting them flow into + # ``log_predict`` would make per-position contributions blow up to + # +inf (``target_clamped * (log(target) - (-inf)) = +inf``). They have + # zero mass under ``target`` (``_ref_attn_full_score`` zeros those + # positions) so their KL contribution should be 0; explicitly mask. + position_valid = torch.isfinite(index_scores) + log_predict = torch.where( + position_valid, index_scores - safe_lse.unsqueeze(-1), torch.zeros_like(index_scores) + ) + contributions = target_clamped * (torch.log(target_clamped) - log_predict) + contributions = torch.where(position_valid, contributions, torch.zeros_like(contributions)) + kl_per_row = contributions.sum(dim=-1) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + return loss_coeff * kl_per_row.mean() + + +def _ref_sparse_attn_forward( + q_flat_bf16: torch.Tensor, # (total_Sq, H, D) + kv_flat_bf16: torch.Tensor, # (total_Skv, D) — K=V, MQA + attn_sink_fp32: torch.Tensor, # (H,) + topk_idxs: torch.Tensor, # (total_Sq, topk) int32, global + softmax_scale: float, + d_v: int, +): + """Pure-PyTorch reference for FlashMLA sparse-attn-fwd output. + + Mirrors the math FlashMLA implements: + * Scores ``S[i, h, k] = Q[i, h] @ K[topk[i, k]]^T * scale`` for valid ``k``; + * Append a per-head sink logit (``attn_sink``); + * ``softmax`` over the (topk + sink) axis; + * ``out[i, h] = sum_k softmax[i, h, k] * V[topk[i, k]]`` (excluding sink). + + Returns ``(out, lse)`` in the same shapes/dtype as the FlashMLA kernel. + Invalid ``-1`` topk entries contribute zero to the softmax (logit -inf). + """ + total_Sq, H, D = q_flat_bf16.shape + topk = topk_idxs.shape[-1] + device = q_flat_bf16.device + q_fp32 = q_flat_bf16.float() + kv_fp32 = kv_flat_bf16.float() + + valid = topk_idxs >= 0 # (total_Sq, topk) + safe = topk_idxs.clamp(min=0).long() + k_gathered = kv_fp32[safe] # (total_Sq, topk, D) + + qk = torch.einsum('ihd,ikd->ihk', q_fp32, k_gathered) * softmax_scale # (Sq, H, topk) + qk = torch.where(valid.unsqueeze(1).expand(-1, H, -1), qk, torch.full_like(qk, float('-inf'))) + sink = attn_sink_fp32.view(1, H, 1).expand(total_Sq, H, 1) # logit + logits = torch.cat([qk, sink], dim=-1) # (Sq, H, topk + 1) + probs = torch.softmax(logits, dim=-1) # numerically stable + probs_kv = probs[..., :topk] # exclude sink contribution from output + + v_gathered = k_gathered # K = V (MQA, head-broadcast) + out_fp32 = torch.einsum('ihk,ikd->ihd', probs_kv, v_gathered) # (Sq, H, D_v=D) + if d_v != D: + out_fp32 = out_fp32[..., :d_v] + out = out_fp32.to(q_flat_bf16.dtype) + + # FlashMLA's KV-only LSE excludes the sink term: + # lse_kv[i, h] = logsumexp_k(qk[i, h, k]) over valid k only. + lse_kv = torch.logsumexp(qk, dim=-1) # (Sq, H), -inf for fully-masked rows + return out, lse_kv + + +# --------------------------------------------------------------------------- +# Score-helper parity tests (sparse + dense): real cuDNN vs PyTorch reference +# --------------------------------------------------------------------------- + + +# Shared small shape across all real-kernel tests to maximize cuDNN compile-cache +# hits. ``ratio=1`` (standard upper-triangular causal) keeps the math simple +# and ensures every row has at least one valid KV position. +_REAL_SHAPES_SPARSE = dict( + b=2, + sq=128, + sk=128, + n_comp=128, + np_=32, + d=512, + idx_nh=32, + idx_hd=128, + # topk = lcm(64, 128) = 128 satisfies SparseScoreRecomputeSm100's + # `topk % n_block_size == 0` (64 for score_type=attention, 128 for indexer). + topk=128, + ratio=1, + softmax_scale=512**-0.5, + indexer_softmax_scale=128**-0.5, +) + + +def _build_real_score_inputs(s, *, with_lse: bool = True, with_topk: bool = True): + """Build a coherent set of bf16 BSHD inputs for the score-helper tests. + + Returns a dict with both bf16 (kernel-ready) and fp32 (reference-math) + views of every tensor, plus optional LSE / topk_indices. + """ + torch.manual_seed(0) + dev = 'cuda' + + q_idx = torch.randn(s['b'], s['sq'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, device=dev) + k_idx = torch.randn(s['b'], s['sk'], s['idx_hd'], dtype=torch.bfloat16, device=dev) + w = torch.randn(s['b'], s['sq'], s['idx_nh'], dtype=torch.bfloat16, device=dev) + + q_attn = torch.randn(s['b'], s['sq'], s['np_'], s['d'], dtype=torch.bfloat16, device=dev) + k_attn = torch.randn(s['b'], s['sk'], s['d'], dtype=torch.bfloat16, device=dev) + + out = dict(q_idx=q_idx, k_idx=k_idx, w=w, q_attn=q_attn, k_attn=k_attn) + + if with_lse: + # LSE = logsumexp(QK*scale, dim=Sk) with the kernel's ratio mask. + # Real LSE input avoids exp(-inf - finite) underflow during reference. + qk = torch.einsum('bqhd,bkd->bqhk', q_attn.float(), k_attn.float()) * s['softmax_scale'] + valid = _ratio_causal_valid_mask(s['sq'], s['sk'], s['ratio'], qk.device).view( + 1, s['sq'], 1, s['sk'] + ) + qk_masked = torch.where(valid, qk, torch.full_like(qk, float('-inf'))) + out['lse'] = torch.logsumexp(qk_masked, dim=-1).clamp(min=-1e30).contiguous() + + if with_topk: + # Pick distinct random valid indices per (b, sq), with a few -1s + # interleaved to exercise the invalid-slot path. + topk = s['topk'] + torch.manual_seed(123) + idxs = torch.randint(0, s['sk'], (s['b'], s['sq'], topk), dtype=torch.int32, device=dev) + # Mark a few slots invalid (-1) to test the topk_indices < 0 path. + invalid = torch.rand(s['b'], s['sq'], topk, device=dev) < 0.1 + idxs = torch.where(invalid, torch.full_like(idxs, -1), idxs) + out['topk'] = idxs + + return out + + +class TestRealKernelScoreHelpers: + """Real-kernel parity tests for the four ``_compute_*`` score helpers + against PyTorch reference implementations. Single parametrized test + covers all four; numeric tolerance is bf16-friendly (raw fp32 score + sums agree to ~5%, normalized distributions to ~5e-3). + """ + + # Each case: (id, sm_min, kernel_name, runner). The runner does the + # call + ref + assertion; it returns nothing on success. + @pytest.mark.parametrize( + "case", + ['sparse_indexer_predict', 'sparse_attn_target', 'dense_indexer_score', 'dense_attn_score'], + ) + def test_real_score_helper(self, case, reset_lazy_kernel_state): + _skip_if_real_kernels_unavailable(sm_min=10) + + s = _REAL_SHAPES_SPARSE + # Each case needs a different combination of the input fixture. + x = _build_real_score_inputs( + s, + with_lse=case.endswith('_attn_target') or case.endswith('_attn_score'), + with_topk=case.startswith('sparse_'), + ) + + from megatron.core.transformer.experimental_attention_variant import dsa_kernels as _dk + + if case == 'sparse_indexer_predict': + # The kernel takes sm_scale=1.0; scale is applied via weights + # pre-multiplication (relu(c·x)·W trick). Reference mirrors that. + scale = s['indexer_softmax_scale'] + w_scaled = (x['w'].float() * scale).to(x['w'].dtype) + out = _dk._compute_indexer_predict( + x['q_idx'], x['k_idx'], w_scaled, x['topk'], qhead_per_kv_head=s['idx_nh'] + ) + ref = _ref_indexer_predict_sparse( + x['q_idx'].float(), x['k_idx'].float(), w_scaled.float(), x['topk'], sm_scale=1.0 + ) + # Softmax outputs in [0, 1]; bf16 element-wise noise can break + # absolute tolerance, so compare directions via cosine similarity. + assert out.shape == ref.shape == (s['b'], s['sq'], s['topk']) + cos = torch.nn.functional.cosine_similarity( + out.flatten().unsqueeze(0).float(), ref.flatten().unsqueeze(0).float() + ).item() + assert cos > 0.99, ( + f"{case}: cos sim = {cos:.4f}, " + f"max abs diff = {(out - ref).abs().max().item():.3e}" + ) + + elif case == 'sparse_attn_target': + out = _dk._compute_attn_target( + x['q_attn'], + x['k_attn'], + x['lse'], + x['topk'], + softmax_scale=s['softmax_scale'], + qhead_per_kv_head=s['np_'], + ) + ref = _ref_attn_target_sparse( + x['q_attn'].float(), + x['k_attn'].float(), + x['lse'], + x['topk'], + softmax_scale=s['softmax_scale'], + ) + assert out.shape == ref.shape == (s['b'], s['sq'], s['topk']) + cos = torch.nn.functional.cosine_similarity( + out.flatten().unsqueeze(0).float(), ref.flatten().unsqueeze(0).float() + ).item() + assert cos > 0.99, ( + f"{case}: cos sim = {cos:.4f}, " + f"max abs diff = {(out - ref).abs().max().item():.3e}" + ) + + elif case == 'dense_indexer_score': + out, denom = _dk._compute_dense_indexer_score( + x['q_idx'], + x['k_idx'].unsqueeze(2), + x['w'], + qhead_per_kv_head=s['idx_nh'], + indexer_softmax_scale=s['indexer_softmax_scale'], + ratio=s['ratio'], + ) + ref_out = _ref_indexer_full_score( + x['q_idx'].float(), + x['k_idx'].float(), + x['w'].float(), + sm_scale=s['indexer_softmax_scale'], + ratio=s['ratio'], + ) + ref_denom = torch.logsumexp(ref_out, dim=-1) + assert out.shape == ref_out.shape == (s['b'], s['sq'], s['sk']) + assert denom.shape == ref_denom.shape == (s['b'], s['sq']) + # Raw fp32 score sums: relative tolerance dominates. Compare + # only valid positions (masked = -inf in both, NaN under sub). + valid = ( + _ratio_causal_valid_mask(s['sq'], s['sk'], s['ratio'], out.device) + .unsqueeze(0) + .expand_as(out) + ) + diff = torch.where(valid, (out - ref_out).abs(), torch.zeros_like(out)) + scale = ref_out.where(valid, torch.zeros_like(ref_out)).abs().max().item() + assert diff.max().item() <= max( + 5e-2, 5e-2 * scale + ), f"{case}: max abs diff = {diff.max().item():.3e}, scale = {scale:.3e}" + row_valid = torch.isfinite(ref_denom) + assert torch.allclose( + denom[row_valid], ref_denom[row_valid], atol=5e-3, rtol=5e-2 + ), f"{case}: LSE max abs diff = {(denom - ref_denom)[row_valid].abs().max().item():.3e}" + + elif case == 'dense_attn_score': + out, denom = _dk._compute_dense_attn_score( + x['q_attn'], + x['k_attn'].unsqueeze(2), + x['lse'], + qhead_per_kv_head=s['np_'], + softmax_scale=s['softmax_scale'], + ratio=s['ratio'], + ) + ref_out = _ref_attn_full_score( + x['q_attn'].float(), + x['k_attn'].float(), + x['lse'], + softmax_scale=s['softmax_scale'], + ratio=s['ratio'], + ) + ref_denom = ref_out.sum(dim=-1) + assert out.shape == ref_out.shape == (s['b'], s['sq'], s['sk']) + assert denom.shape == ref_denom.shape == (s['b'], s['sq']) + valid = ( + _ratio_causal_valid_mask(s['sq'], s['sk'], s['ratio'], out.device) + .unsqueeze(0) + .expand_as(out) + ) + diff = torch.where(valid, (out - ref_out).abs(), torch.zeros_like(out)) + # exp(QK*scale - LSE) outputs in (0, ~1]: absolute dominates. + assert diff.max().item() <= 5e-3, f"{case}: max abs diff = {diff.max().item():.3e}" + assert torch.allclose(denom, ref_denom, atol=5e-3, rtol=5e-2), ( + f"{case}: denom max abs diff = " f"{(denom - ref_denom).abs().max().item():.3e}" + ) + + else: + raise AssertionError(f"unknown case: {case}") + + +# --------------------------------------------------------------------------- +# KL loss reference parity (dense path; sparse already CPU-tested above). +# --------------------------------------------------------------------------- + + +class TestRealKernelKLLossDense: + """End-to-end parity for ``_kl_loss_from_dense_scores``: run the real + cuDNN dense score kernels, feed their outputs into the helper, and + compare the KL value to the all-PyTorch reference. + """ + + @pytest.mark.parametrize("dummy", [None]) + def test_real_dense_kl_loss_matches_reference(self, dummy, reset_lazy_kernel_state): + _skip_if_real_kernels_unavailable(sm_min=10) + from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + _compute_dense_attn_score, + _compute_dense_indexer_score, + _kl_loss_from_dense_scores, + ) + + s = _REAL_SHAPES_SPARSE + x = _build_real_score_inputs(s, with_lse=True, with_topk=False) + loss_coeff = 0.5 + + index_score, index_lse = _compute_dense_indexer_score( + x['q_idx'], + x['k_idx'].unsqueeze(2), + x['w'], + qhead_per_kv_head=s['idx_nh'], + indexer_softmax_scale=s['indexer_softmax_scale'], + ratio=s['ratio'], + ) + attn_score, attn_l1norm = _compute_dense_attn_score( + x['q_attn'], + x['k_attn'].unsqueeze(2), + x['lse'], + qhead_per_kv_head=s['np_'], + softmax_scale=s['softmax_scale'], + ratio=s['ratio'], + ) + + loss_actual = _kl_loss_from_dense_scores( + attn_score, attn_l1norm, index_score, index_lse, loss_coeff + ) + loss_ref = _ref_dense_indexer_loss( + x['q_idx'].float(), + x['k_idx'].float(), + x['w'].float(), + x['q_attn'].float(), + x['k_attn'].float(), + x['lse'], + indexer_softmax_scale=s['indexer_softmax_scale'], + attn_softmax_scale=s['softmax_scale'], + ratio=s['ratio'], + loss_coeff=loss_coeff, + ) + assert torch.allclose(loss_actual, loss_ref, atol=1e-3, rtol=1e-2), ( + f"actual = {loss_actual.item():.6f}, ref = {loss_ref.item():.6f}, " + f"abs diff = {(loss_actual - loss_ref).abs().item():.3e}" + ) + + +# --------------------------------------------------------------------------- +# Real ``indexer_topk``: the top-K set should match the reference ranking. +# --------------------------------------------------------------------------- + + +class TestRealKernelIndexerTopk: + """Real-kernel parity for :func:`indexer_topk`: the SET of selected + top-K indices must match a PyTorch reference ranking. We compare sets + rather than ordered lists because BF16 ties may be broken differently. + """ + + @pytest.mark.parametrize("dummy", [None]) + def test_real_indexer_topk_set_matches_reference(self, dummy, reset_lazy_kernel_state): + _skip_if_real_kernels_unavailable(sm_min=10) # IndexerForward is SM100+ + from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + indexer_topk, + ) + + # IndexerForward requires idx_hd=128 and qhpkv in (32, 64). Use an + # SBHD shape that matches what csa.py produces (tensors are SBHD, + # ratio is the indexer's compression ratio). b=2 exercises the + # batch-aware ``seq_lens.repeat(b)`` and the ``(b*sq, sk) → (b, sq, + # topk)`` reshape inside ``_indexer_topk_bshd``. + s = dict( + b=2, + sq=128, + idx_nh=32, + idx_hd=128, + sk=128, + indexer_topk=8, + ratio=4, + indexer_softmax_scale=128**-0.5, + ) + torch.manual_seed(0) + dev = 'cuda' + q_indexer = torch.randn( + s['sq'], s['b'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, device=dev + ) + k_indexer = torch.randn(s['sk'], s['b'], s['idx_hd'], dtype=torch.bfloat16, device=dev) + weights = torch.randn(s['sq'], s['b'], s['idx_nh'], dtype=torch.bfloat16, device=dev) + + topk_indices, topk_length = indexer_topk( + q_indexer, + k_indexer, + weights, + topk=s['indexer_topk'], + ratio=s['ratio'], + indexer_softmax_scale=s['indexer_softmax_scale'], + ) + assert topk_indices.shape == (s['b'], s['sq'], s['indexer_topk']) + assert topk_indices.dtype == torch.int32 + + # Reference: full indexer score, ratio causal mask, take top-K per row + # by descending score. Score is sm_scale * sum_h ReLU(Q@K) * W. + q_bshd = q_indexer.permute(1, 0, 2, 3).contiguous().float() + k_bsd = k_indexer.permute(1, 0, 2).contiguous().float() + w_bsh = weights.permute(1, 0, 2).contiguous().float() + ref_scores = _ref_indexer_full_score( + q_bshd, k_bsd, w_bsh, sm_scale=s['indexer_softmax_scale'], ratio=s['ratio'] + ) # (B, Sq, Sk), -inf at masked positions + + # For each row, count valid positions (un-masked). topk_length should + # match min(indexer_topk, num_valid). + n_valid = (ref_scores > float('-inf')).sum(dim=-1) # (B, Sq) + expected_length = n_valid.clamp(max=s['indexer_topk']).int() + assert torch.equal(topk_length, expected_length) + + # Set comparison row-by-row. Skip rows with 0 valid (kernel returns + # all -1; reference picks arbitrary -inf positions). + ref_topk = torch.topk(ref_scores, k=s['indexer_topk'], dim=-1).indices + for bi in range(s['b']): + for qi in range(s['sq']): + n = int(expected_length[bi, qi].item()) + if n == 0: + # Kernel must report all -1. + assert torch.all(topk_indices[bi, qi] == -1) + continue + actual_set = set(topk_indices[bi, qi, :n].tolist()) + ref_set = set(ref_topk[bi, qi, :n].tolist()) + # BF16 ties may differ: allow up to ~10% mismatch on small K. + inter = actual_set & ref_set + assert len(inter) >= max(1, n - 1), ( + f"row (b={bi}, q={qi}): " + f"actual {sorted(actual_set)} vs ref {sorted(ref_set)}" + ) + + +# --------------------------------------------------------------------------- +# Real ``dsa_sparse_attn``: forward + backward parity vs PyTorch reference. +# --------------------------------------------------------------------------- + + +class TestRealKernelDsaSparseAttn: + """Real-kernel parity for :func:`dsa_sparse_attn`. Forward uses real + FlashMLA + the SBHD/flat reshape wrapper; backward uses the real cuDNN + sparse-attn-bwd kernel. Both checked in one test against the + pure-PyTorch sparse-attn reference (``_ref_sparse_attn_forward``). + """ + + SHAPES = dict(b=2, sq=128, np_=64, d=512, skv=128, topk=32, softmax_scale=512**-0.5) + + def _make_inputs(self, *, requires_grad: bool): + s = self.SHAPES + torch.manual_seed(0) + dev = 'cuda' + + def make_leaf(*shape, dtype): + t = torch.randn(*shape, dtype=dtype, device=dev) + return t.detach().clone().requires_grad_(True) if requires_grad else t + + query = make_leaf(s['sq'], s['b'], s['np_'], s['d'], dtype=torch.bfloat16) + kv = make_leaf(s['skv'], s['b'], s['d'], dtype=torch.bfloat16) + attn_sink = torch.zeros(s['np_'], dtype=torch.float32, device=dev) + if requires_grad: + attn_sink = attn_sink.detach().clone().requires_grad_(True) + + # Coherent valid global topk indices in SBHD-flat layout, with a + # standard causal mask (index <= q_idx). + torch.manual_seed(1) + topk_local = torch.randint( + 0, s['skv'], (s['b'], s['sq'], s['topk']), dtype=torch.int64, device=dev + ) + q_idx = torch.arange(s['sq'], device=dev).view(1, -1, 1) + topk_local = torch.minimum(topk_local, q_idx) + global_idxs = local_to_global_flat(topk_local, s['b'], s['skv']).contiguous() + return query, kv, attn_sink, global_idxs + + def test_real_dsa_sparse_attn_fwd_bwd_matches_reference(self, reset_lazy_kernel_state): + """Forward output AND backward gradients (dq, dkv, d_sink) must + match a pure-PyTorch sparse-attn reference. Combining both checks + in one test halves cuDNN compile time vs running them separately, + since they share the same kernel cache key. + """ + _skip_if_real_kernels_unavailable(sm_min=10, need_flash_mla=True) + s = self.SHAPES + + # ---- Real path: forward + backward via dsa_sparse_attn ---- + query, kv, attn_sink, global_idxs = self._make_inputs(requires_grad=True) + out = dsa_sparse_attn(query, kv, attn_sink, global_idxs, softmax_scale=s['softmax_scale']) + torch.manual_seed(7) + upstream = torch.randn_like(out) + (out * upstream).sum().backward() + dq_actual = query.grad.float().clone() + dkv_actual = kv.grad.float().clone() + dsink_actual = attn_sink.grad.float().clone() + out_actual = out.float().detach().clone() + + # ---- Reference: pure-PyTorch forward + autograd ---- + query_ref, kv_ref, attn_sink_ref, _ = self._make_inputs(requires_grad=True) + q_flat = query_ref.reshape(s['sq'] * s['b'], s['np_'], s['d']) + kv_flat = kv_ref.reshape(s['skv'] * s['b'], s['d']) + ref_out_flat, _ = _ref_sparse_attn_forward( + q_flat, + kv_flat, + attn_sink_ref, + global_idxs, + softmax_scale=s['softmax_scale'], + d_v=s['d'], + ) + ref_out = ref_out_flat.reshape(s['sq'], s['b'], s['np_'], s['d']).reshape( + s['sq'], s['b'], s['np_'] * s['d'] + ) + (ref_out * upstream).sum().backward() + + # ---- Forward + backward parity (cos sim) ---- + # bf16 GEMM accumulators in FlashMLA fwd / cuDNN sparse-attn-bwd + # make element-wise tolerances brittle (esp. dkv); compare each + # tensor's direction via cosine similarity instead. + def _cos(a, b): + return torch.nn.functional.cosine_similarity( + a.flatten().unsqueeze(0).float(), b.flatten().unsqueeze(0).float() + ).item() + + assert out_actual.shape == ref_out.shape + for name, actual, ref in [ + ('forward', out_actual, ref_out.float()), + ('dq', dq_actual, query_ref.grad.float()), + ('dkv', dkv_actual, kv_ref.grad.float()), + ('d_sink', dsink_actual, attn_sink_ref.grad.float()), + ]: + cos = _cos(actual, ref) + assert cos > 0.99, ( + f"{name}: cos sim = {cos:.4f}, " + f"max abs diff = {(actual - ref).abs().max().item():.3e}" + ) + + +# --------------------------------------------------------------------------- +# Real ``fused_indexer_sparse_attn``: dense-loss path end-to-end parity. +# --------------------------------------------------------------------------- + + +class TestRealKernelFusedIndexerSparseAttn: + """End-to-end parity for the dense loss path of + :func:`fused_indexer_sparse_attn`: real cuDNN dense kernels (forward + + backward) + real FlashMLA, compared to ``_ref_dense_indexer_loss``. + + Backward grad correctness for the indexer-grad kernel is established by + ``TestRealKernelKLLossDense`` (kernel-level math) and + ``TestDenseFusedIndexerSparseAttn::test_dense_backward_calls_dense_indexer_grad`` + (mock-based plumbing). This class only checks the loss SCALAR value. + """ + + # FlashMLA only accepts indexer_topk ∈ {0, 512, 1024, 2048} and a limited + # set of h_q values (np_=64 is the supported one used by the sibling + # DsaSparseAttn real-kernel test). n_comp must be ≥ indexer_topk; skv ≥ + # n_comp so kv_offset = skv - n_comp > 0 still exercises the offset path. + SHAPES = dict( + b=2, + sq=128, + np_=64, + d=512, + skv=640, + n_comp=512, + # cudnn DSA (dense_)indexer_backward kernels require heads >= 64. + idx_nh=64, + idx_hd=128, + indexer_topk=512, + ratio=4, + win_topk=8, + softmax_scale=512**-0.5, + indexer_softmax_scale=128**-0.5, + ) + + def test_real_fused_dense_loss_matches_reference(self, reset_lazy_kernel_state): + """Real dense path's KL loss value matches the all-PyTorch reference + on the same inputs. The reference uses an analytical + ``logsumexp(QK*scale, ratio mask)`` for ``lse_indexer`` (FlashMLA + emits its own internal lse_indexer that differs slightly), so the + tolerance is wider than for the kernel-only ``KLLossDense`` test. + """ + _skip_if_real_kernels_unavailable(sm_min=10, need_flash_mla=True) + s = self.SHAPES + torch.manual_seed(0) + dev = 'cuda' + loss_coeff = 0.5 + + # Build inputs once; share between actual and reference. + query = torch.randn(s['sq'], s['b'], s['np_'], s['d'], dtype=torch.bfloat16, device=dev) + kv_full = torch.randn(s['skv'], s['b'], s['d'], dtype=torch.bfloat16, device=dev) + attn_sink = torch.zeros(s['np_'], dtype=torch.float32, device=dev) + torch.manual_seed(1) + win_idxs = torch.randint( + 0, s['sq'], (s['b'], s['sq'], s['win_topk']), dtype=torch.int32, device=dev + ) + q_indexer = torch.randn( + s['sq'], s['b'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, device=dev + ) + k_indexer = torch.randn(s['n_comp'], s['b'], s['idx_hd'], dtype=torch.bfloat16, device=dev) + weights = torch.randn(s['sq'], s['b'], s['idx_nh'], dtype=torch.bfloat16, device=dev) + kv_offset = s['skv'] - s['n_comp'] + + # Real path. + _, indexer_loss = fused_indexer_sparse_attn( + query, + kv_full, + attn_sink, + win_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk=s['indexer_topk'], + ratio=s['ratio'], + softmax_scale=s['softmax_scale'], + indexer_softmax_scale=s['indexer_softmax_scale'], + loss_coeff=loss_coeff, + sparse_loss=False, + kv_offset=kv_offset, + ) + + # Reference: SBHD->BSHD once, build analytical lse_ref, compute KL. + q_idx_bshd = q_indexer.permute(1, 0, 2, 3).contiguous().float() + k_idx_bsd = k_indexer.permute(1, 0, 2).contiguous().float() + w_bsh = weights.permute(1, 0, 2).contiguous().float() + q_attn_bshd = query.permute(1, 0, 2, 3).contiguous().float() + k_attn_bsd = kv_full[kv_offset:].permute(1, 0, 2).contiguous().float() + + # PyTorch reference that mirrors the fused path's dense-loss math + # exactly. Two non-obvious requirements: + # * Use FlashMLA's emitted ``lse_indexer`` (logsumexp over the + # indexer-selected top-K positions, with the per-head sink term), + # not an analytical full-KV logsumexp. Otherwise the per-row LSE + # basis differs from the kernel by ~50x. + # * Do NOT apply the ratio-causal mask in the reference scores — + # the dense-score-recompute kernels emit values at every position + # (no internal masking). Masking the reference would shift the + # ``attn_score / attn_l1norm`` normalization and the indexer LSE + # basis, producing a different KL than the kernel's. + from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + _dsa_fwd_flash_mla, + _indexer_topk_bshd, + _kl_loss_from_dense_scores, + _sbhd_to_bshd_indexer_inputs, + ) + + # Run indexer + FlashMLA to capture the same ``lse_indexer`` the fused + # path consumes internally. + effective_topk = min(s['indexer_topk'], s['n_comp']) + q_idx_bshd_bf, k_idx_bsd_bf, _, w_bsh_scaled_bf = _sbhd_to_bshd_indexer_inputs( + q_indexer, k_indexer, weights, s['indexer_softmax_scale'] + ) + topk_indices_cmp, _, _ = _indexer_topk_bshd( + q_idx_bshd_bf, k_idx_bsd_bf, w_bsh_scaled_bf, effective_topk, s['ratio'] + ) + compress_topk_idxs = torch.where(topk_indices_cmp >= 0, topk_indices_cmp + kv_offset, -1) + combined_local = torch.cat([compress_topk_idxs, win_idxs], dim=-1) + global_idxs = local_to_global_flat(combined_local, s['b'], s['skv']) + q_flat = query.reshape(s['sq'] * s['b'], s['np_'], s['d']) + kv_flat = kv_full.reshape(s['skv'] * s['b'], s['d']) + _, _, lse_indexer = _dsa_fwd_flash_mla( + q_flat, + kv_flat, + global_idxs, + s['softmax_scale'], + attn_sink=attn_sink, + topk_length=None, + indexer_topk=effective_topk, + ) + lse_indexer_bsqh = lse_indexer.reshape(s['sq'], s['b'], s['np_']).permute(1, 0, 2) + + # Attention path: exp(QK*scale - lse_indexer), head-summed. No mask. + qk_attn = torch.einsum('bqhd,bkd->bqhk', q_attn_bshd, k_attn_bsd) * s['softmax_scale'] + attn_score_ref = torch.exp(qk_attn - lse_indexer_bsqh.unsqueeze(-1)).sum(dim=2) + attn_l1norm_ref = attn_score_ref.sum(dim=-1) + + # Indexer path: ReLU(QK_indexer) * W head-summed, scaled by + # ``indexer_softmax_scale`` once. The fused path applies the scale + # via pre-scaled ``w_bsh_scaled`` only (not again inside the + # kernel), giving ``score = sum_h(relu(QK) * w_raw) * sm_scale``. + qk_idx = torch.einsum('bqhd,bkd->bqhk', q_idx_bshd, k_idx_bsd) + idx_score_ref = (torch.relu(qk_idx) * w_bsh.unsqueeze(-1)).sum(dim=2) * s[ + 'indexer_softmax_scale' + ] + idx_lse_ref = torch.logsumexp(idx_score_ref, dim=-1) + + loss_ref = _kl_loss_from_dense_scores( + attn_score_ref, attn_l1norm_ref, idx_score_ref, idx_lse_ref, loss_coeff + ) + assert torch.allclose(indexer_loss, loss_ref, atol=5e-2, rtol=1e-1), ( + f"actual = {indexer_loss.item():.6f}, ref = {loss_ref.item():.6f}, " + f"abs diff = {(indexer_loss - loss_ref).abs().item():.3e}" + ) + + +# --------------------------------------------------------------------------- +# Real-kernel dense-indexer backward parity (kernel vs autograd) +# --------------------------------------------------------------------------- + + +class TestRealKernelDenseIndexerBackward: + """End-to-end gradient parity for cuDNN ``dense_indexer_backward_wrapper``. + + The kernel-level dense forward (``dense_attn_score_recompute_wrapper`` / + ``dense_indexer_score_recompute_wrapper``) is covered by + :class:`TestRealKernelKLLossDense` (forward value only), and the + fused-path dense forward by + :class:`TestRealKernelFusedIndexerSparseAttn` (forward loss scalar only). + Nothing else in this file exercises the dense backward kernel against + autograd at real-kernel scale — the only other coverage is the + mock-based plumbing tests in :class:`TestDenseFusedIndexerSparseAttn`. + + This class runs the real fused dense-loss path (``sparse_loss=False``), + calls ``.backward()`` on the indexer loss to obtain kernel-emitted + gradients for ``q_indexer`` / ``k_indexer`` / ``weights``, then compares + each against PyTorch autograd through the matching analytical + ``_kl_loss_from_dense_scores`` formulation. ``attn_score`` / + ``attn_l1norm`` / ``lse_indexer`` are captured from the kernel and + treated as constants on the reference side (the attention-side + backward is a separate kernel, out of scope here). + + Mirrors the gradient parity check done by + ``test_dsv4_hybrid_native_parity::test_dsv4_hybrid_attention_matches_native_reference`` + but at the indexer-tensor level — so the same kernel discrepancy is + reproducible without spinning up the full DSv4 hybrid layer. + """ + + SHAPES = TestRealKernelFusedIndexerSparseAttn.SHAPES + + def test_real_dense_backward_grad_matches_autograd(self, reset_lazy_kernel_state): + _skip_if_real_kernels_unavailable(sm_min=10, need_flash_mla=True) + s = self.SHAPES + torch.manual_seed(0) + dev = 'cuda' + loss_coeff = 0.5 + + # Shared, non-grad context tensors. + query = torch.randn(s['sq'], s['b'], s['np_'], s['d'], dtype=torch.bfloat16, device=dev) + kv_full = torch.randn(s['skv'], s['b'], s['d'], dtype=torch.bfloat16, device=dev) + attn_sink = torch.zeros(s['np_'], dtype=torch.float32, device=dev) + torch.manual_seed(1) + win_idxs = torch.randint( + 0, s['sq'], (s['b'], s['sq'], s['win_topk']), dtype=torch.int32, device=dev + ) + q_idx_init = torch.randn( + s['sq'], s['b'], s['idx_nh'], s['idx_hd'], dtype=torch.bfloat16, device=dev + ) + k_idx_init = torch.randn(s['n_comp'], s['b'], s['idx_hd'], dtype=torch.bfloat16, device=dev) + w_init = torch.randn(s['sq'], s['b'], s['idx_nh'], dtype=torch.bfloat16, device=dev) + kv_offset = s['skv'] - s['n_comp'] + + # ---- Actual: real fused path; capture kernel-emitted indexer grads. + q_idx_real = q_idx_init.detach().clone().requires_grad_(True) + k_idx_real = k_idx_init.detach().clone().requires_grad_(True) + w_real = w_init.detach().clone().requires_grad_(True) + _, indexer_loss = fused_indexer_sparse_attn( + query, + kv_full, + attn_sink, + win_idxs, + q_idx_real, + k_idx_real, + w_real, + indexer_topk=s['indexer_topk'], + ratio=s['ratio'], + softmax_scale=s['softmax_scale'], + indexer_softmax_scale=s['indexer_softmax_scale'], + loss_coeff=loss_coeff, + sparse_loss=False, + kv_offset=kv_offset, + ) + indexer_loss.backward() + dq_kernel = q_idx_real.grad.detach().clone() + dk_kernel = k_idx_real.grad.detach().clone() + dw_kernel = w_real.grad.detach().clone() + + # ---- Reference: capture the kernel's attn-side / lse_indexer (treated + # as constants) and run autograd through the analytical dense KL. + from megatron.core.transformer.experimental_attention_variant.dsa_kernels import ( + _compute_dense_attn_score, + _dsa_fwd_flash_mla, + _indexer_topk_bshd, + _kl_loss_from_dense_scores, + _sbhd_to_bshd_indexer_inputs, + ) + + effective_topk = min(s['indexer_topk'], s['n_comp']) + with torch.no_grad(): + q_idx_bshd_bf, k_idx_bsd_bf, _, w_bsh_scaled_bf = _sbhd_to_bshd_indexer_inputs( + q_idx_init, k_idx_init, w_init, s['indexer_softmax_scale'] + ) + topk_indices_cmp, _, _ = _indexer_topk_bshd( + q_idx_bshd_bf, k_idx_bsd_bf, w_bsh_scaled_bf, effective_topk, s['ratio'] + ) + compress_topk_idxs = torch.where( + topk_indices_cmp >= 0, topk_indices_cmp + kv_offset, -1 + ) + combined_local = torch.cat([compress_topk_idxs, win_idxs], dim=-1) + global_idxs = local_to_global_flat(combined_local, s['b'], s['skv']) + q_flat = query.reshape(s['sq'] * s['b'], s['np_'], s['d']) + kv_flat = kv_full.reshape(s['skv'] * s['b'], s['d']) + _, _, lse_indexer = _dsa_fwd_flash_mla( + q_flat, + kv_flat, + global_idxs, + s['softmax_scale'], + attn_sink=attn_sink, + topk_length=None, + indexer_topk=effective_topk, + ) + lse_indexer_bsqh = lse_indexer.reshape(s['sq'], s['b'], s['np_']).permute(1, 0, 2) + + q_attn_bshd = query.permute(1, 0, 2, 3).contiguous() + k_attn_bsd = kv_full[kv_offset:].permute(1, 0, 2).contiguous() + attn_score_const, attn_l1norm_const = _compute_dense_attn_score( + q_attn_bshd, + k_attn_bsd.unsqueeze(2), + lse_indexer_bsqh, + qhead_per_kv_head=s['np_'], + softmax_scale=s['softmax_scale'], + ratio=s['ratio'], + ) + + # Autograd reference: same dense-KL formula, but index_score / + # index_lse depend on q_idx_ref / k_idx_ref / w_ref so autograd can + # propagate gradients back to them. + # + # Stop-gradient alignment: the reference uses the kernel's actual + # bf16-emitted ``indexer_scores`` as the forward value (so the + # loss numerics match exactly), but routes the chain rule through + # the analytical fp32 score. Without this, fp32-analytical vs + # bf16-kernel forward scores drift ~0.4% and contaminate the + # backward comparison with a forward-side artifact (~0.7% cosine + # gap) that has nothing to do with the backward kernel itself. + with torch.no_grad(): + q_idx_bshd_k, k_idx_bsd_k, _, w_bsh_scaled_k = _sbhd_to_bshd_indexer_inputs( + q_idx_init, k_idx_init, w_init, s['indexer_softmax_scale'] + ) + _, _, kernel_indexer_scores = _indexer_topk_bshd( + q_idx_bshd_k, k_idx_bsd_k, w_bsh_scaled_k, effective_topk, s['ratio'] + ) + + q_idx_ref = q_idx_init.detach().clone().requires_grad_(True) + k_idx_ref = k_idx_init.detach().clone().requires_grad_(True) + w_ref = w_init.detach().clone().requires_grad_(True) + q_idx_bshd_ref = q_idx_ref.permute(1, 0, 2, 3).contiguous().float() + k_idx_bsd_ref = k_idx_ref.permute(1, 0, 2).contiguous().float() + w_bsh_ref = w_ref.permute(1, 0, 2).contiguous().float() + qk_idx = torch.einsum('bqhd,bkd->bqhk', q_idx_bshd_ref, k_idx_bsd_ref) + idx_score_analytical = (torch.relu(qk_idx) * w_bsh_ref.unsqueeze(-1)).sum(dim=2) * s[ + 'indexer_softmax_scale' + ] + idx_score_aligned = ( + idx_score_analytical + (kernel_indexer_scores - idx_score_analytical).detach() + ) + idx_lse_aligned = torch.logsumexp(idx_score_aligned, dim=-1) + loss_ref = _kl_loss_from_dense_scores( + attn_score_const.detach(), + attn_l1norm_const.detach(), + idx_score_aligned, + idx_lse_aligned, + loss_coeff, + ) + loss_ref.backward() + + def cosine(a, b): + return torch.nn.functional.cosine_similarity( + a.flatten().double().unsqueeze(0), b.flatten().double().unsqueeze(0) + ).item() + + # eps=5e-4 covers the residual bf16↔autograd precision noise on + # d_q (~2.6e-4 observed at this scale); d_k and d_weights agree + # to within ~1e-5 / exact respectively. Tighten if the kernel's + # dense backward improves or if d_q noise drops. + eps = 5e-4 + for name, dk_grad, dr_grad in [ + ('d q_indexer', dq_kernel, q_idx_ref.grad), + ('d k_indexer', dk_kernel, k_idx_ref.grad), + ('d weights', dw_kernel, w_ref.grad), + ]: + cs = cosine(dk_grad, dr_grad) + assert cs > 1 - eps, f"{name}: cosine_sim={cs:.10f}, eps={eps}" + + +# --------------------------------------------------------------------------- +# Public surface +# --------------------------------------------------------------------------- + + +class TestPublicApi: + """The ``__all__`` list documents the public surface; verify that every + advertised symbol is importable, that the public free functions are + callable, and that the autograd Functions inherit from the right base. + """ + + def test_public_surface(self): + from megatron.core.transformer.experimental_attention_variant import dsa_kernels + + for name in dsa_kernels.__all__: + assert hasattr(dsa_kernels, name), f"__all__ lists {name!r} but it is missing" + + for fn in ( + build_flat_topk_idxs, + local_to_global_flat, + dsa_sparse_attn, + indexer_topk, + fused_indexer_sparse_attn, + ): + assert callable(fn) + + assert issubclass(SparseAttnFunc, torch.autograd.Function) + assert issubclass(FusedIndexerSparseAttnFunc, torch.autograd.Function) diff --git a/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_attention.py b/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_attention.py index dec153682cc..9dafa013c4d 100644 --- a/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_attention.py +++ b/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_attention.py @@ -584,3 +584,84 @@ def test_hash_moe_layer_requires_input_ids_but_hca_layer_does_not(self): assert context is None assert output.shape == hidden.shape assert torch.isfinite(output).all() + + +# =========================================================================== +# apply_rope_fusion tests +# =========================================================================== + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.skipif(not HAVE_TE, reason="transformer_engine not available") +class TestDSv4HybridRopeFusion: + """Test that apply_rope_fusion=True works for both yarn and non-yarn layers. + + DSv4 Hybrid uses YarnRotaryEmbedding for layers with compress_ratio > 1 + and standard RotaryEmbedding for layers with compress_ratio <= 1. The + fused RoPE path must obtain cos/sin from both embedding classes via + get_cached_cos_sin. + + compress_ratios=[0, 4, 128, 4]: layer 1 has ratio 0 (standard + RotaryEmbedding), layers 2-4 have ratio > 1 (YarnRotaryEmbedding). + """ + + @pytest.fixture(scope='class', autouse=True) + def setup_method(self, request): + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, pipeline_model_parallel_size=1 + ) + torch.manual_seed(_SEED) + model_parallel_cuda_manual_seed(_SEED) + + cls = request.cls + cls.pg = ProcessGroupCollection.use_mpu_process_groups() + + yield + Utils.destroy_model_parallel() + + def test_rope_fusion_forward_backward_parity(self): + """Fused RoPE forward/backward succeeds and matches the unfused path.""" + seq_len = 128 + batch_size = 2 + + torch.manual_seed(_SEED) + model_parallel_cuda_manual_seed(_SEED) + fused_config = _make_config(apply_rope_fusion=True) + attn_fused = _build_attention(fused_config, layer_number=4, pg_collection=self.pg).cuda() + attn_fused.train() + + torch.manual_seed(_SEED) + model_parallel_cuda_manual_seed(_SEED) + unfused_config = _make_config(apply_rope_fusion=False) + attn_unfused = _build_attention( + unfused_config, layer_number=4, pg_collection=self.pg + ).cuda() + attn_unfused.train() + + hidden = torch.randn( + seq_len, batch_size, fused_config.hidden_size, dtype=torch.bfloat16 + ).cuda() + + out_fused, _ = attn_fused(hidden_states=hidden, attention_mask=None) + out_unfused, _ = attn_unfused(hidden_states=hidden, attention_mask=None) + + assert out_fused.shape == (seq_len, batch_size, fused_config.hidden_size) + assert torch.isfinite(out_fused).all() + # Production code forces ``mscale=1.0`` (DSv4 contract) in both + # fused and unfused paths, so the only residual is bf16 noise from + # the fused Triton kernel's different accumulation order vs the + # PyTorch eager ops. The residual concentrates at output positions + # whose values are near zero (sign flips on tiny magnitudes drive + # the worst-case max-abs-diff). + torch.testing.assert_close(out_fused, out_unfused, atol=3e-2, rtol=3e-2) + + hidden_fused = hidden.detach().clone().requires_grad_(True) + hidden_unfused = hidden.detach().clone().requires_grad_(True) + + attn_fused(hidden_states=hidden_fused, attention_mask=None)[0].sum().backward() + attn_unfused(hidden_states=hidden_unfused, attention_mask=None)[0].sum().backward() + + assert hidden_fused.grad is not None + for name, param in attn_fused.named_parameters(): + if param.requires_grad: + assert param.grad is not None, f"No gradient for parameter {name}" diff --git a/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_native_parity.py b/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_native_parity.py new file mode 100644 index 00000000000..72f37489a67 --- /dev/null +++ b/tests/unit_tests/transformer/experimental_attention_variant/test_dsv4_hybrid_native_parity.py @@ -0,0 +1,951 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +import gc +import math + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F + +from megatron.core.extensions.transformer_engine import HAVE_TE +from megatron.core.extensions.transformer_engine_spec_provider import TESpecProvider +from megatron.core.models.gpt.experimental_attention_variant_module_specs import ( + get_dsv4_hybrid_module_spec_for_backend, +) +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.experimental_attention_variant.dsa import DSAIndexerLossAutoScaler +from megatron.core.transformer.spec_utils import build_module +from megatron.core.transformer.transformer_config import MLATransformerConfig +from megatron.core.utils import init_method_normal, scaled_init_method_normal +from tests.unit_tests.test_utilities import Utils + +_SEED = 1234 +# Fused-path eps: in this test ``apply_rope_fusion`` is coupled to +# ``apply_dsa_kernel_fusion``, so the fused branch exercises BOTH the +# cudnn DSA kernels AND the Triton fused MLA RoPE kernel. The MLA RoPE +# kernel's bf16 numerics differ from pytorch eager RoPE by ~2-3e-3 cosine +# at the input-gradient level after propagating through the layer; that +# noise dominates the original 1.5e-4 DSA-kernel-only budget. Empirical +# worst case observed: ``cosine_sim ≈ 0.998`` on hidden_grad / upstream +# param grads, ``≈ 0.9995`` on the forward output. +_FUSED_SIMILARITY_EPS = 3e-3 +_UNFUSED_SIMILARITY_EPS = 3e-5 +# ``core_attention.attn_sink`` is a per-head scalar bias whose gradient +# is just the sum of the sink's softmax probability over all positions +# (no spatial averaging). Tiny shape + no averaging means the per-element +# fused-rope drift accumulates directly into the grad rather than washing +# out, so its parity floor is roughly an order of magnitude looser than +# the per-token gradients. Empirical worst case ``cosine_sim ≈ 0.984``. +_FUSED_ATTN_SINK_GRAD_SIMILARITY_EPS = 2e-2 +# Fused dense-loss path: ``dense_indexer_backward_wrapper`` consumes raw +# scores plus L1-norm/LSE separately (not pre-softmaxed distributions, as +# the sparse variant does), so the kernel-vs-autograd precision noise is +# not absorbed by a softmax boundary. Combined with TE-vs-nn linear/RoPE +# drift on q_indexer/k_indexer/weights, the indexer param-grad cosine sim +# floors around 1e-3 here. Applied only to ``.indexer.`` params when +# ``apply_dsa_kernel_fusion=True`` and ``dsa_indexer_use_sparse_loss=False``. +# Kept distinct from ``_FUSED_SIMILARITY_EPS`` so the per-param branch +# stays readable, even though both currently sit in the same order. +_FUSED_DENSE_INDEXER_GRAD_SIMILARITY_EPS = 3e-3 + + +@torch.compile +def _native_q_rms_norm(query: torch.Tensor, eps: float) -> torch.Tensor: + return query * torch.rsqrt(query.square().mean(-1, keepdim=True) + eps) + + +_DSV4_VARIANTS = { + "flash": { + "hidden_size": 4096, + "num_attention_heads": 64, + "q_lora_rank": 1024, + "v_head_dim": 512, + "qk_pos_emb_head_dim": 64, + "o_groups": 8, + "o_lora_rank": 1024, + "csa_compress_rotary_base": 40000, + "dsa_indexer_topk": 512, + }, + "pro": { + "hidden_size": 7168, + "num_attention_heads": 128, + "q_lora_rank": 1536, + "v_head_dim": 512, + "qk_pos_emb_head_dim": 64, + "o_groups": 16, + "o_lora_rank": 1024, + "csa_compress_rotary_base": 160000, + "dsa_indexer_topk": 1024, + }, +} + +_DSA_BACKENDS = [ + pytest.param("fused", True, id="fused"), + pytest.param("unfused", False, id="unfused"), +] + + +def _make_config( + variant: str, + compress_ratio: int, + apply_dsa_kernel_fusion: bool = False, + calculate_per_token_loss: bool = False, + dsa_indexer_use_sparse_loss: bool = False, +) -> MLATransformerConfig: + shape = _DSV4_VARIANTS[variant] + mcore_ratio = 0 if compress_ratio == 1 else compress_ratio + qk_head_dim = shape["v_head_dim"] - shape["qk_pos_emb_head_dim"] + config = MLATransformerConfig( + multi_latent_attention=True, + experimental_attention_variant="dsv4_hybrid", + num_layers=1, + hidden_size=shape["hidden_size"], + num_attention_heads=shape["num_attention_heads"], + q_lora_rank=shape["q_lora_rank"], + kv_lora_rank=qk_head_dim, + qk_head_dim=qk_head_dim, + qk_pos_emb_head_dim=shape["qk_pos_emb_head_dim"], + v_head_dim=shape["v_head_dim"], + o_groups=shape["o_groups"], + o_lora_rank=shape["o_lora_rank"], + csa_compress_ratios=[mcore_ratio], + csa_window_size=128, + csa_dense_mode=False, + dsa_indexer_n_heads=64, + dsa_indexer_head_dim=128, + dsa_indexer_topk=shape["dsa_indexer_topk"], + dsa_indexer_loss_coeff=0.01, + dsa_indexer_use_sparse_loss=dsa_indexer_use_sparse_loss, + calculate_per_token_loss=calculate_per_token_loss, + add_bias_linear=False, + bf16=True, + params_dtype=torch.bfloat16, + layernorm_epsilon=1e-6, + normalization="RMSNorm", + qk_layernorm=True, + layernorm_zero_centered_gamma=False, + expert_model_parallel_size=1, + tensor_model_parallel_size=1, + sequence_parallel=False, + context_parallel_size=1, + rope_type="yarn" if apply_dsa_kernel_fusion else "rope", + rotary_base=10000, + rotary_percent=1.0, + csa_compress_rotary_base=shape["csa_compress_rotary_base"], + recompute_granularity=None, + recompute_modules=[], + fine_grained_activation_offloading=False, + gradient_accumulation_fusion=False, + fp8=False, + fp4=False, + init_method=init_method_normal(0.02), + output_layer_init_method=scaled_init_method_normal(0.02, 1, multiplier=2.0), + kv_channels=shape["v_head_dim"], + num_query_groups=shape["num_attention_heads"], + batch_invariant_mode=False, + cache_mla_latents=False, + use_cpu_initialization=True, + perform_initialization=True, + symmetric_ar_type=None, + disable_parameter_transpose_cache=False, + init_model_with_meta_device=False, + delay_wgrad_compute=False, + tp_comm_overlap=False, + softmax_scale=None, + apply_dsa_kernel_fusion=apply_dsa_kernel_fusion, + apply_rope_fusion=apply_dsa_kernel_fusion, + ) + return config + + +def _precompute_freqs_cis( + dim: int, + seqlen: int, + device, + base: float, + *, + original_seq_len: int = 0, + factor: float = 1.0, + beta_fast: float = 32.0, + beta_slow: float = 1.0, +) -> torch.Tensor: + """Precompute the [seq, 1, 1, dim] freqs table used by ``_apply_rotary_emb``. + + Matches the golden DSv4 reference (``Megatron-LM/model.py:precompute_freqs_cis``) + and ``YarnRotaryEmbedding`` semantics: + + * ``original_seq_len > 0`` enables YaRN frequency interpolation between + the ``beta_fast`` / ``beta_slow`` correction-range bounds. Frequencies + below the low boundary are divided by ``factor`` (interpolation); above + the high boundary, freqs pass through (extrapolation); a smooth linear + ramp blends the two in between. + * ``original_seq_len == 0`` reverts to plain RoPE with no scaling — the + window-only branch on the production side. + """ + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) + if original_seq_len > 0: + + def _correction_dim(num_rotations): + return (dim * math.log(original_seq_len / (num_rotations * 2 * math.pi))) / ( + 2 * math.log(base) + ) + + low = max(int(math.floor(_correction_dim(beta_fast))), 0) + high = min(int(math.ceil(_correction_dim(beta_slow))), dim - 1) + if low == high: + high += 1 # avoid div-by-zero in the ramp + ramp = (torch.arange(dim // 2, dtype=torch.float32, device=device) - low) / (high - low) + smooth = 1.0 - torch.clamp(ramp, 0.0, 1.0) + freqs = freqs / factor * (1.0 - smooth) + freqs * smooth + + t = torch.arange(seqlen, device=device) + freqs = torch.outer(t, freqs) + return torch.cat((freqs, freqs), dim=-1)[:, None, None, :] + + +def _apply_rotary_emb( + x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False +) -> torch.Tensor: + if x.numel() == 0: + return x + freqs = freqs_cis.to(x.device) + if freqs.dim() == x.dim() + 1 and freqs.size(-2) == 1: + freqs = freqs.squeeze(-2) + + rot_dim = freqs.size(-1) + x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:] + x1 = x_rot[..., 0::2] + x2 = x_rot[..., 1::2] + x_rot = torch.cat((x1, x2), dim=-1) + + cos = torch.cos(freqs).to(x_rot.dtype) + sin = torch.sin(freqs).to(x_rot.dtype) + if inverse: + sin = -sin + + rot_half_1, rot_half_2 = torch.chunk(x_rot, 2, dim=-1) + x_rotated = torch.cat((-rot_half_2, rot_half_1), dim=-1) + out = (x_rot * cos) + (x_rotated * sin) + + x1, x2 = torch.chunk(out, 2, dim=-1) + out = torch.stack((x1, x2), dim=-1).flatten(start_dim=-2) + return torch.cat((out, x_pass), dim=-1) + + +def _native_hadamard_transform(x: torch.Tensor) -> torch.Tensor: + n = x.size(-1) + if n <= 0 or n & (n - 1): + raise ValueError(f"Hadamard transform requires power-of-two last dim, got {n}") + dtype = x.dtype + y = x.float() + shape = y.shape + h = 1 + while h < n: + y = y.reshape(*shape[:-1], -1, 2, h) + a = y[..., 0, :] + b = y[..., 1, :] + y = torch.cat((a + b, a - b), dim=-1) + h *= 2 + return (y.reshape(shape) * (n**-0.5)).to(dtype) + + +def _get_window_topk_idxs( + window_size: int, batch_size: int, seqlen: int, device: torch.device +) -> torch.Tensor: + base = torch.arange(seqlen, device=device).unsqueeze(1) + offsets = torch.arange(window_size, device=device) + matrix = (base - window_size + 1).clamp(min=0) + offsets + matrix = torch.where(matrix > base, -1, matrix) + return matrix.unsqueeze(0).expand(batch_size, -1, -1) + + +def _get_compress_topk_idxs( + ratio: int, batch_size: int, seqlen: int, offset: int, device: torch.device +) -> torch.Tensor: + n_compressed = seqlen // ratio + matrix = torch.arange(n_compressed, device=device).repeat(seqlen, 1) + mask = matrix >= torch.arange(1, seqlen + 1, device=device).unsqueeze(1) // ratio + matrix = torch.where(mask, -1, matrix + offset) + return matrix.unsqueeze(0).expand(batch_size, -1, -1) + + +def _native_sparse_attn( + query: torch.Tensor, + kv_full: torch.Tensor, + attn_sink: torch.Tensor, + topk_indices: torch.Tensor, + softmax_scale: float, +) -> torch.Tensor: + sq, batch_size, num_heads, head_dim = query.size() + kv_t = kv_full.permute(1, 0, 2) + safe_indices = topk_indices.clamp(min=0).long() + gather_index = safe_indices.unsqueeze(-1).expand(-1, -1, -1, head_dim) + kv_gathered = torch.gather(kv_t.unsqueeze(1).expand(-1, sq, -1, -1), dim=2, index=gather_index) + + q = query.permute(1, 2, 0, 3).float() + scores = torch.einsum("bnsh,bskh->bnsk", q, kv_gathered.float()) * softmax_scale + scores = scores.masked_fill((topk_indices < 0).unsqueeze(1), float("-inf")) + + sink = attn_sink.view(1, num_heads, 1, 1).float() + scores_max = torch.max(scores.max(dim=-1, keepdim=True).values, sink) + exp_scores = torch.exp(scores - scores_max) + exp_sink = torch.exp(sink - scores_max) + attn_weights = exp_scores / (exp_scores.sum(dim=-1, keepdim=True) + exp_sink) + + output = torch.einsum("bnsk,bskh->bnsh", attn_weights, kv_gathered.float()) + output = output.to(query.dtype).permute(2, 0, 1, 3).contiguous() + return output.reshape(sq, batch_size, num_heads * head_dim) + + +def _native_fused_sparse_indexer_loss( + index_scores: torch.Tensor, + topk_indices: torch.Tensor, + query: torch.Tensor, + compressed_kv: torch.Tensor, + attn_sink: torch.Tensor, + softmax_scale: float, + loss_coeff: float, + sparse_loss: bool, + calculate_per_token_loss: bool, +) -> torch.Tensor: + batch_size, seqlen, _ = topk_indices.size() + num_heads, head_dim = query.size(2), query.size(3) + n_compressed = compressed_kv.size(0) + + sink = attn_sink.detach().view(1, num_heads, 1, 1).float() + q = query.detach().permute(1, 2, 0, 3).float() + compressed_kv_t = compressed_kv.detach().permute(1, 0, 2) + + if sparse_loss: + safe_indices = topk_indices.clamp(min=0).long() + valid = topk_indices >= 0 + row_valid = valid.any(dim=-1, keepdim=True) + + predict_logits = torch.gather(index_scores, dim=-1, index=safe_indices) + predict_logits = predict_logits.masked_fill(~valid, float("-inf")) + predict_logits = predict_logits.masked_fill(~row_valid, 0.0) + predict = F.softmax(predict_logits, dim=-1, dtype=torch.float32) + predict = predict * row_valid.float() + + selected_kv = torch.gather( + compressed_kv_t.unsqueeze(1).expand(-1, seqlen, -1, -1), + dim=2, + index=safe_indices.unsqueeze(-1).expand(-1, -1, -1, head_dim), + ) + attn_scores = torch.einsum("bhsd,bskd->bhsk", q, selected_kv.float()) + attn_scores = attn_scores * softmax_scale + attn_scores = attn_scores.masked_fill(~valid.unsqueeze(1), float("-inf")) + else: + # Dense loss: KL is computed over the FULL compressed-KV axis (not + # just topk). Index-side and attention-side both use the kernel's + # ratio-causal mask, which we derive analytically from the + # compress_ratio (= seqlen / n_compressed): position k of the + # compressed-KV is valid for query row q iff k < (q + 1) // ratio. + compress_ratio = seqlen // n_compressed + k_idx = torch.arange(n_compressed, device=index_scores.device) + valid_per_q = ( + torch.arange(1, seqlen + 1, device=index_scores.device) // compress_ratio + ).clamp(max=n_compressed) + finite_pos = k_idx.view(1, 1, -1) < valid_per_q.view(1, -1, 1) # (1, sq, n_compressed) + finite_pos = finite_pos.expand(batch_size, -1, -1) + row_valid = finite_pos.any(dim=-1, keepdim=True) + + predict_logits = index_scores.masked_fill(~finite_pos, float("-inf")) + predict_logits = predict_logits.masked_fill(~row_valid, 0.0) + predict = F.softmax(predict_logits, dim=-1, dtype=torch.float32) + predict = predict * row_valid.float() + + attn_scores = torch.einsum("bhsd,bkd->bhsk", q, compressed_kv_t.float()) + attn_scores = attn_scores * softmax_scale + attn_mask = finite_pos.unsqueeze(1).expand(-1, num_heads, -1, -1) + attn_scores = attn_scores.masked_fill(~attn_mask, float("-inf")) + + score_max = torch.max(attn_scores.max(dim=-1, keepdim=True).values, sink) + exp_scores = torch.exp(attn_scores - score_max) + exp_sink = torch.exp(sink - score_max) + attn_probs = exp_scores / (exp_scores.sum(dim=-1, keepdim=True) + exp_sink) + target = attn_probs.sum(dim=1) + target = target / target.sum(dim=-1, keepdim=True).clamp(min=1e-10) + target = target * row_valid.float() + + eps = torch.finfo(torch.float32).tiny + target = target.clamp(min=eps) + predict = predict.clamp(min=eps) + kl_per_row = (target * (torch.log(target) - torch.log(predict))).sum(dim=-1) + kl_per_row = torch.where(row_valid.squeeze(-1), kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss_coeff * loss + + +def _native_unfused_sparse_indexer_loss( + index_scores: torch.Tensor, + topk_indices: torch.Tensor, + query: torch.Tensor, + compressed_kv: torch.Tensor, + softmax_scale: float, + loss_coeff: float, + sparse_loss: bool, + causal_mask: torch.Tensor, + calculate_per_token_loss: bool, +) -> torch.Tensor: + sq, batch_size, num_heads, _ = query.size() + sk = compressed_kv.size(0) + mask = causal_mask.to(dtype=torch.float32) + + attention_scores = torch.einsum( + "sbhd,tbd->bhst", query.detach().float(), compressed_kv.detach().float() + ) + attention_scores = attention_scores * softmax_scale + attention_scores = attention_scores + mask.view(batch_size, 1, sq, sk) + index_scores = index_scores + mask + + if sparse_loss: + index_mask = torch.full( + (batch_size, sq, sk), float("-inf"), dtype=torch.float32, device=index_scores.device + ).scatter_(-1, topk_indices.clamp(min=0), 0) + attention_scores = attention_scores + index_mask.view(batch_size, 1, sq, sk) + index_scores = index_scores + index_mask + + row_valid = (mask > float("-inf")).any(dim=-1) + attn_row_mask = row_valid.view(batch_size, 1, sq, 1) + idx_row_mask = row_valid.view(batch_size, sq, 1) + + attention_scores = attention_scores.masked_fill(~attn_row_mask, 0.0) + index_scores = index_scores.masked_fill(~idx_row_mask, 0.0) + + attention_probs = F.softmax(attention_scores, dim=-1, dtype=torch.float32) + predict = F.softmax(index_scores, dim=-1, dtype=torch.float32) + attention_probs = attention_probs * attn_row_mask.float() + predict = predict * idx_row_mask.float() + + target = attention_probs.sum(dim=1) + target = target / target.sum(dim=-1, keepdim=True) + eps = torch.finfo(torch.float32).tiny + target = target.clamp(min=eps) + predict = predict.clamp(min=eps) + kl_per_row = (target * (torch.log(target) - torch.log(predict))).sum(dim=-1) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss * loss_coeff + + +class NativeCompressor(nn.Module): + def __init__( + self, config: MLATransformerConfig, compress_ratio: int, head_dim: int, rotate: bool + ): + super().__init__() + self.compress_ratio = compress_ratio + self.head_dim = head_dim + self.overlap = compress_ratio == 4 + self.coff = 1 + int(self.overlap) + self.rotate = rotate + self.qk_pos_emb_head_dim = config.qk_pos_emb_head_dim + self.rope_base = ( + config.csa_compress_rotary_base if compress_ratio > 1 else config.rotary_base + ) + # YaRN frequency interpolation is enabled only for compressed sequences + # (matches ``DSv4HybridAttention``'s ``use_compressed_yarn = ratio > 1``). + if compress_ratio > 1: + self._rope_yarn_kwargs = dict( + original_seq_len=config.original_max_position_embeddings, + factor=config.rotary_scaling_factor, + beta_fast=config.beta_fast, + beta_slow=config.beta_slow, + ) + else: + self._rope_yarn_kwargs = dict() + + self.linear_wkv = nn.Linear(config.hidden_size, self.coff * head_dim, bias=False) + self.linear_wgate = nn.Linear(config.hidden_size, self.coff * head_dim, bias=False) + self.ape = nn.Parameter( + torch.empty(compress_ratio, self.coff * head_dim, dtype=torch.float32) + ) + self.norm = nn.RMSNorm(head_dim, eps=config.layernorm_epsilon) + + def _overlap_transform(self, tensor: torch.Tensor, fill_value: float = 0) -> torch.Tensor: + n_groups, ratio, batch_size, _ = tensor.size() + new_tensor = tensor.new_full((n_groups, 2 * ratio, batch_size, self.head_dim), fill_value) + new_tensor[:, ratio:] = tensor[:, :, :, self.head_dim :] + new_tensor[1:, :ratio] = tensor[:-1, :, :, : self.head_dim] + return new_tensor + + def forward(self, x: torch.Tensor) -> torch.Tensor | None: + sq, batch_size, _ = x.size() + ratio = self.compress_ratio + if sq < ratio: + return None + + kv = self.linear_wkv(x) + score = self.linear_wgate(x) + + cutoff = (sq // ratio) * ratio + kv = kv[:cutoff] + score = score[:cutoff] + n_compressed = cutoff // ratio + + kv = kv.view(n_compressed, ratio, batch_size, -1) + score = score.view(n_compressed, ratio, batch_size, -1) + score = score + self.ape.view(1, ratio, 1, -1) + + if self.overlap: + kv = self._overlap_transform(kv, fill_value=0) + score = self._overlap_transform(score, fill_value=float("-inf")) + + kv = (kv * torch.softmax(score, dim=1)).sum(dim=1) + kv = self.norm(kv.to(x.dtype)) + + pos_dim = self.qk_pos_emb_head_dim + content, rotary = torch.split(kv, [self.head_dim - pos_dim, pos_dim], dim=-1) + freqs_cis = _precompute_freqs_cis( + pos_dim, + n_compressed * ratio, + device=x.device, + base=self.rope_base, + **self._rope_yarn_kwargs, + ) + freqs_cis = freqs_cis[: n_compressed * ratio : ratio][:n_compressed] + rotary = _apply_rotary_emb(rotary, freqs_cis) + kv = torch.cat([content, rotary], dim=-1) + + if self.rotate: + kv = _native_hadamard_transform(kv) + return kv + + +class NativeCSAIndexer(nn.Module): + def __init__(self, config: MLATransformerConfig, compress_ratio: int): + super().__init__() + self.compress_ratio = compress_ratio + self.index_n_heads = config.dsa_indexer_n_heads + self.index_head_dim = config.dsa_indexer_head_dim + self.index_topk = config.dsa_indexer_topk + self.qk_pos_emb_head_dim = config.qk_pos_emb_head_dim + self.softmax_scale = self.index_head_dim**-0.5 + self.apply_dsa_kernel_fusion = config.apply_dsa_kernel_fusion + self.rope_base = config.csa_compress_rotary_base + # CSA indexer is only instantiated for ``compress_ratio == 4``, which is + # always the YaRN-enabled branch on the production side. + self._rope_yarn_kwargs = dict( + original_seq_len=config.original_max_position_embeddings, + factor=config.rotary_scaling_factor, + beta_fast=config.beta_fast, + beta_slow=config.beta_slow, + ) + + self.linear_wq_b = nn.Linear( + config.q_lora_rank, self.index_n_heads * self.index_head_dim, bias=False + ) + self.linear_weights_proj = nn.Linear(config.hidden_size, self.index_n_heads, bias=False) + self.compressor = NativeCompressor( + config=config, compress_ratio=compress_ratio, head_dim=self.index_head_dim, rotate=True + ) + + def forward_before_topk( + self, x: torch.Tensor, qr: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + sq, batch_size, _ = x.size() + q = self.linear_wq_b(qr).view(sq, batch_size, self.index_n_heads, self.index_head_dim) + pos_dim = self.qk_pos_emb_head_dim + q_content, q_rotary = torch.split(q, [self.index_head_dim - pos_dim, pos_dim], dim=-1) + freqs_cis = _precompute_freqs_cis( + pos_dim, sq, device=x.device, base=self.rope_base, **self._rope_yarn_kwargs + ) + q_rotary = _apply_rotary_emb(q_rotary, freqs_cis) + q = _native_hadamard_transform(torch.cat([q_content, q_rotary], dim=-1)) + + k = self.compressor(x) + weights = self.linear_weights_proj(x) * (self.index_n_heads**-0.5) + return q, k, weights + + def forward( + self, x: torch.Tensor, qr: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q, k, weights = self.forward_before_topk(x, qr) + weights_scaled = weights.float() * self.softmax_scale + if self.apply_dsa_kernel_fusion: + weights_scaled = weights_scaled.to(weights.dtype).float() + scores = torch.einsum("sbhd,tbd->sbht", q.float(), k.float()) + scores = torch.relu(scores) * weights_scaled.unsqueeze(-1) + scores = scores.sum(dim=2).transpose(0, 1) + + sq = x.size(0) + n_compressed = k.size(0) + valid_per_query = ( + torch.arange(1, sq + 1, device=x.device).unsqueeze(0) // self.compress_ratio + ).clamp(max=n_compressed) + invalid = torch.arange(n_compressed, device=x.device).view( + 1, 1, -1 + ) >= valid_per_query.unsqueeze(-1) + scores = scores.masked_fill(invalid.expand_as(scores), float("-inf")) + + topk = min(self.index_topk, n_compressed) + topk_scores, topk_indices = scores.topk(topk, dim=-1) + topk_indices = torch.where(topk_scores.isneginf(), -1, topk_indices) + return q, k, weights, scores, topk_indices + + +class NativeCompressedSparseAttention(nn.Module): + def __init__(self, config: MLATransformerConfig, compress_ratio: int): + super().__init__() + self.compress_ratio = compress_ratio + self.window_size = config.csa_window_size + self.num_heads = config.num_attention_heads + self.head_dim = config.v_head_dim + self.softmax_scale = self.head_dim**-0.5 + self.indexer_loss_coeff = config.dsa_indexer_loss_coeff + self.indexer_use_sparse_loss = config.dsa_indexer_use_sparse_loss + self.calculate_per_token_loss = config.calculate_per_token_loss + self.apply_dsa_kernel_fusion = config.apply_dsa_kernel_fusion + + self.attn_sink = nn.Parameter(torch.zeros(self.num_heads, dtype=torch.float32)) + self.compressor = ( + NativeCompressor( + config=config, compress_ratio=compress_ratio, head_dim=self.head_dim, rotate=False + ) + if compress_ratio > 1 + else None + ) + self.indexer = NativeCSAIndexer(config, compress_ratio) if compress_ratio == 4 else None + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + x: torch.Tensor, + qr: torch.Tensor, + pg_collection, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + sq, batch_size, _, _ = query.size() + kv = key.squeeze(-2) + n_compressed = 0 + + if self.compressor is not None: + compressed_kv = self.compressor(x) + if compressed_kv is not None: + kv_full = torch.cat([kv, compressed_kv], dim=0) + n_compressed = compressed_kv.size(0) + else: + kv_full = kv + else: + compressed_kv = None + kv_full = kv + + window_idxs = _get_window_topk_idxs(self.window_size, batch_size, sq, query.device) + indexer_loss = None + if self.compress_ratio > 1 and n_compressed > 0: + offset = sq + if self.indexer is not None: + q_idx, k_idx, weights_idx, index_scores, topk_compressed = self.indexer( + x.detach(), qr.detach() + ) + topk_compressed_for_attn = torch.where( + topk_compressed >= 0, topk_compressed + offset, -1 + ) + + if not self.apply_dsa_kernel_fusion: + causal_mask = ( + torch.arange(n_compressed, device=x.device).unsqueeze(0).expand(sq, -1) + ) + positions = torch.arange(1, sq + 1, device=x.device).unsqueeze(1) + causal_mask = ( + torch.where( + causal_mask >= positions // self.compress_ratio, float("-inf"), 0.0 + ) + .unsqueeze(0) + .expand(batch_size, -1, -1) + ) + indexer_loss = _native_unfused_sparse_indexer_loss( + index_scores, + topk_compressed, + query.detach(), + compressed_kv.detach(), + self.softmax_scale, + self.indexer_loss_coeff, + self.indexer_use_sparse_loss, + causal_mask, + self.calculate_per_token_loss, + ) + else: + indexer_loss = _native_fused_sparse_indexer_loss( + index_scores, + topk_compressed, + query, + compressed_kv, + self.attn_sink, + self.softmax_scale, + self.indexer_loss_coeff, + self.indexer_use_sparse_loss, + self.calculate_per_token_loss, + ) + else: + topk_compressed_for_attn = _get_compress_topk_idxs( + self.compress_ratio, batch_size, sq, offset, query.device + ) + if self.indexer is not None and self.apply_dsa_kernel_fusion: + topk_idxs = torch.cat([topk_compressed_for_attn, window_idxs], dim=-1) + else: + topk_idxs = torch.cat([window_idxs, topk_compressed_for_attn], dim=-1) + else: + topk_idxs = window_idxs + + output = _native_sparse_attn(query, kv_full, self.attn_sink, topk_idxs, self.softmax_scale) + return output, indexer_loss + + +class NativeDSv4HybridAttention(nn.Module): + def __init__(self, config: MLATransformerConfig, compress_ratio: int): + super().__init__() + self.config = config + self.compress_ratio = compress_ratio + self.num_heads = config.num_attention_heads + self.head_dim = config.v_head_dim + self.pos_dim = config.qk_pos_emb_head_dim + self.nope_dim = config.v_head_dim - config.qk_pos_emb_head_dim + self.rope_base = ( + config.csa_compress_rotary_base if compress_ratio > 1 else config.rotary_base + ) + if compress_ratio > 1: + self._rope_yarn_kwargs = dict( + original_seq_len=config.original_max_position_embeddings, + factor=config.rotary_scaling_factor, + beta_fast=config.beta_fast, + beta_slow=config.beta_slow, + ) + else: + self._rope_yarn_kwargs = dict() + + self.linear_q_down_proj = nn.Linear(config.hidden_size, config.q_lora_rank, bias=False) + self.q_layernorm = nn.RMSNorm(config.q_lora_rank, eps=config.layernorm_epsilon) + self.linear_q_up_proj = nn.Linear( + config.q_lora_rank, config.num_attention_heads * config.v_head_dim, bias=False + ) + self.linear_kv_proj = nn.Linear(config.hidden_size, config.v_head_dim, bias=False) + self.kv_layernorm = nn.RMSNorm(config.v_head_dim, eps=config.layernorm_epsilon) + self.core_attention = NativeCompressedSparseAttention(config, compress_ratio) + group_in = (config.num_attention_heads * config.v_head_dim) // config.o_groups + self.linear_o_group_proj = nn.Parameter( + torch.empty(config.o_groups * config.o_lora_rank, group_in) + ) + self.linear_proj = nn.Linear( + config.o_groups * config.o_lora_rank, config.hidden_size, bias=False + ) + + def forward( + self, hidden_states: torch.Tensor, pg_collection + ) -> tuple[torch.Tensor, torch.Tensor | None]: + sq, batch_size, _ = hidden_states.size() + freqs_cis = _precompute_freqs_cis( + self.pos_dim, sq, hidden_states.device, self.rope_base, **self._rope_yarn_kwargs + ) + + qr = self.q_layernorm(self.linear_q_down_proj(hidden_states)) + query = self.linear_q_up_proj(qr).view(sq, batch_size, self.num_heads, self.head_dim) + query = _native_q_rms_norm(query, self.config.layernorm_epsilon) + q_content, q_rotary = torch.split(query, [self.nope_dim, self.pos_dim], dim=-1) + query = torch.cat([q_content, _apply_rotary_emb(q_rotary, freqs_cis)], dim=-1) + + key = self.kv_layernorm(self.linear_kv_proj(hidden_states)) + k_content, k_rotary = torch.split(key, [self.nope_dim, self.pos_dim], dim=-1) + key = torch.cat([k_content, _apply_rotary_emb(k_rotary, freqs_cis)], dim=-1) + key = key.unsqueeze(-2) + + core_out, indexer_loss = self.core_attention( + query=query, key=key, x=hidden_states, qr=qr, pg_collection=pg_collection + ) + + core_out = core_out.view(sq, batch_size, self.num_heads, self.head_dim) + out_content, out_rotary = torch.split(core_out, [self.nope_dim, self.pos_dim], dim=-1) + core_out = torch.cat( + [out_content, _apply_rotary_emb(out_rotary, freqs_cis, inverse=True)], dim=-1 + ) + core_out = core_out.view(sq, batch_size, -1) + + core_out = core_out.view(sq, batch_size, self.config.o_groups, -1) + wo_a = self.linear_o_group_proj.view(self.config.o_groups, self.config.o_lora_rank, -1) + core_out = torch.einsum("...gd,grd->...gr", core_out, wo_a) + core_out = core_out.reshape(sq, batch_size, -1) + return self.linear_proj(core_out), indexer_loss + + +def _cosine_sim(a: torch.Tensor, b: torch.Tensor) -> float: + return F.cosine_similarity( + a.flatten().double().unsqueeze(0), b.flatten().double().unsqueeze(0) + ).item() + + +def _tensor_sim(a: torch.Tensor, b: torch.Tensor) -> float: + a, b = a.double(), b.double() + denom = (a * a + b * b).sum() + return (2.0 * (a * b).sum() / denom).item() if denom else 1.0 + + +def _assert_similarity(a: torch.Tensor, b: torch.Tensor, label: str, eps: float): + assert torch.isfinite(a).all() + assert torch.isfinite(b).all() + cosine_sim = _cosine_sim(a, b) + tensor_sim = _tensor_sim(a, b) + assert cosine_sim > 1 - eps, f"{label}: cosine_sim={cosine_sim:.10f}, eps={eps}" + assert tensor_sim > 1 - eps, f"{label}: tensor_sim={tensor_sim:.10f}, eps={eps}" + + +def _copy_real_params_to_native(real_layer: nn.Module, native_layer: nn.Module): + real_params = dict(real_layer.named_parameters()) + for name, native_param in native_layer.named_parameters(): + assert name in real_params, f"Missing real parameter for native parameter {name}" + real_param = real_params[name] + assert ( + native_param.shape == real_param.shape + ), f"Shape mismatch for {name}: native={native_param.shape}, real={real_param.shape}" + native_param.data = real_param.data.to( + device=native_param.device, dtype=real_param.dtype + ).clone() + return real_params + + +def _skip_if_real_kernels_unavailable(*, sm_min: int = 9, need_flash_mla: bool = False): + """Pytest-side gate for real-kernel tests. Raises ``pytest.skip`` if + any of the runtime dependencies are missing. + """ + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + sm_major = torch.cuda.get_device_capability()[0] + if sm_major < sm_min: + pytest.skip(f"requires SM{sm_min}+, found SM{sm_major}") + cudnn = pytest.importorskip("cudnn") + from packaging.version import Version + + if Version(cudnn.__version__) < Version("1.24.0"): + pytest.skip(f"requires cudnn>=1.24.0, found {cudnn.__version__}") + if not hasattr(cudnn, 'DSA'): + pytest.skip("cudnn.DSA namespace not available") + if need_flash_mla: + pytest.importorskip("flash_mla") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.skipif(not HAVE_TE, reason="transformer_engine not available") +class TestDSv4HybridNativeParity: + + @classmethod + def setup_class(cls): + Utils.initialize_model_parallel(tensor_model_parallel_size=1, context_parallel_size=1) + + @classmethod + def teardown_class(cls): + Utils.destroy_model_parallel() + + def setup_method(self): + DSAIndexerLossAutoScaler.main_loss_backward_scale = None + torch.manual_seed(_SEED) + torch.cuda.manual_seed(_SEED) + model_parallel_cuda_manual_seed(_SEED) + + def teardown_method(self): + gc.collect() + torch.cuda.empty_cache() + + @pytest.mark.parametrize(("backend", "apply_dsa_kernel_fusion"), _DSA_BACKENDS) + @pytest.mark.parametrize("variant", ["flash", "pro"]) + @pytest.mark.parametrize("compress_ratio", [1, 4, 128]) + @pytest.mark.parametrize( + ("seqlen", "calculate_per_token_loss", "dsa_indexer_use_sparse_loss"), + [ + (4096, False, False), + (4096, False, True), + (4096, True, False), + (4096, True, True), + (8192, True, True), + ], + ) + def test_attention_matches_native_reference( + self, + variant: str, + compress_ratio: int, + seqlen: int, + backend: str, + apply_dsa_kernel_fusion: bool, + calculate_per_token_loss: bool, + dsa_indexer_use_sparse_loss: bool, + ): + if apply_dsa_kernel_fusion: + _skip_if_real_kernels_unavailable(sm_min=10) + major, _ = torch.cuda.get_device_capability() + if major < 10 and not apply_dsa_kernel_fusion and seqlen > 4096: + pytest.skip("seqlen > 4096 may OOM on Hopper with unfused DSA implementation") + + config = _make_config( + variant, + compress_ratio, + apply_dsa_kernel_fusion=apply_dsa_kernel_fusion, + calculate_per_token_loss=calculate_per_token_loss, + dsa_indexer_use_sparse_loss=dsa_indexer_use_sparse_loss, + ) + similarity_eps = ( + _UNFUSED_SIMILARITY_EPS if not apply_dsa_kernel_fusion else _FUSED_SIMILARITY_EPS + ) + pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=["tp", "cp"]) + spec = get_dsv4_hybrid_module_spec_for_backend(config=config, backend=TESpecProvider()) + + mcore_ratio = 0 if compress_ratio == 1 else compress_ratio + real_layer = build_module( + spec, config=config, layer_number=1, cp_comm_type=None, pg_collection=pg_collection + ).cuda() + native_layer = NativeDSv4HybridAttention(config, mcore_ratio).cuda() + real_params = _copy_real_params_to_native(real_layer, native_layer) + + bsz = 1 + for _ in range(1): + hidden_states = torch.randn( + seqlen, + bsz, + config.hidden_size, + dtype=torch.bfloat16, + device="cuda", + requires_grad=True, + ) + hidden_states_native = hidden_states.detach().clone().requires_grad_(True) + grad = torch.randn_like(hidden_states) + + real_out, _ = real_layer(hidden_states=hidden_states, attention_mask=None) + native_out, native_indexer_loss = native_layer(hidden_states_native, pg_collection) + + _assert_similarity( + real_out.detach(), + native_out.detach(), + f"{backend}-{variant}-{compress_ratio}-{seqlen}:out", + eps=similarity_eps, + ) + + real_out.backward(grad) + native_out.backward(grad) + if native_indexer_loss is not None: + native_indexer_loss.backward() + + _assert_similarity( + hidden_states.grad, + hidden_states_native.grad, + f"{backend}-{variant}-{compress_ratio}-{seqlen}:hidden_grad", + eps=similarity_eps, + ) + + is_fused_dense = apply_dsa_kernel_fusion and not dsa_indexer_use_sparse_loss + for name, native_param in native_layer.named_parameters(): + real_param = real_params[name] + if compress_ratio != 4 and ".indexer." in name: + continue + assert native_param.grad is not None, f"Missing native grad for {name}" + assert real_param.grad is not None, f"Missing real grad for {name}" + if apply_dsa_kernel_fusion and "core_attention.attn_sink" in name: + param_eps = _FUSED_ATTN_SINK_GRAD_SIMILARITY_EPS + elif is_fused_dense and ".indexer." in name: + param_eps = _FUSED_DENSE_INDEXER_GRAD_SIMILARITY_EPS + else: + param_eps = similarity_eps + _assert_similarity( + real_param.grad, + native_param.grad, + f"{backend}-{variant}-{compress_ratio}-{seqlen}:param_grad:{name}", + eps=param_eps, + ) diff --git a/tests/unit_tests/transformer/moe/test_token_dispatcher.py b/tests/unit_tests/transformer/moe/test_token_dispatcher.py index dead2c0c12d..af1c4f924f9 100644 --- a/tests/unit_tests/transformer/moe/test_token_dispatcher.py +++ b/tests/unit_tests/transformer/moe/test_token_dispatcher.py @@ -2,16 +2,24 @@ import copy import dataclasses +import math from types import SimpleNamespace import pytest import torch from megatron.core import config, parallel_state -from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_local_submodules +from megatron.core.extensions.transformer_engine import get_thd_partitioned_indices +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_local_submodules, + get_gpt_layer_with_transformer_engine_spec, +) +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.moe.fused_a2a import reset_hybrid_ep_buffer from megatron.core.transformer.moe.moe_layer import MoELayer from megatron.core.transformer.moe.moe_utils import get_capacity from megatron.core.transformer.moe.token_dispatcher import MoETokenDispatcher +from megatron.core.transformer.transformer_block import TransformerBlock from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.typed_torch import apply_module from megatron.core.utils import is_te_min_version @@ -461,6 +469,171 @@ def is_hybrid_ep_available(): return HAVE_HYBRIDEP +def _round_up(value, divisor): + return value if divisor <= 1 else (value + divisor - 1) // divisor * divisor + + +def _get_thd_padded_seqlens(seqlens, cp_size, tp_size): + # This follows the runtime packed-sequence path used by the Moonlight script: + # per-sequence lengths must be CP partitionable, and the packed token count + # must be even for TP/SP slicing. + cp_divisor = 2 * cp_size if cp_size > 1 else 1 + padded_seqlens = [_round_up(seqlen, cp_divisor) for seqlen in seqlens] + total_seqlen = sum(padded_seqlens) + total_alignment = math.lcm(cp_divisor, tp_size) + padded_seqlens[-1] += _round_up(total_seqlen, total_alignment) - total_seqlen + return padded_seqlens + + +def _to_cu_seqlens(seqlens): + cu_seqlens = torch.empty(len(seqlens) + 1, dtype=torch.int32, device="cuda") + cu_seqlens[0] = 0 + cu_seqlens[1:] = torch.cumsum(torch.tensor(seqlens, dtype=torch.int32, device="cuda"), dim=0) + return cu_seqlens + + +def _make_thd_packed_seq_params(seqlens, cp_size, tp_size): + padded_seqlens = _get_thd_padded_seqlens(seqlens, cp_size, tp_size) + cu_seqlens_padded = _to_cu_seqlens(padded_seqlens) + max_seqlen = max(padded_seqlens) + # Match get_batch_on_this_rank_for_sequence_packing(): TE consumes padded + # cumulative lengths as both cu_seqlens and cu_seqlens_padded for THD. + return PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_padded, + cu_seqlens_kv=cu_seqlens_padded, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + ) + + +def _make_sharded_thd_hidden_states(seqlens, hidden_size, cp_size, tp_size, dtype): + padded_seqlens = _get_thd_padded_seqlens(seqlens, cp_size, tp_size) + padded_sequences = [] + for seqlen, padded_seqlen in zip(seqlens, padded_seqlens): + sequence = torch.randn(seqlen, hidden_size, device="cuda", dtype=dtype) + if padded_seqlen > seqlen: + sequence = torch.cat( + [ + sequence, + torch.zeros(padded_seqlen - seqlen, hidden_size, device="cuda", dtype=dtype), + ], + dim=0, + ) + padded_sequences.append(sequence) + + hidden_states = torch.cat(padded_sequences, dim=0) + if cp_size > 1: + cu_seqlens_padded = _to_cu_seqlens(padded_seqlens) + cp_rank = parallel_state.get_context_parallel_rank() + index = get_thd_partitioned_indices( + cu_seqlens_padded, hidden_states.shape[0], cp_size, cp_rank + ) + hidden_states = hidden_states.index_select(0, index) + + tp_rank = parallel_state.get_tensor_model_parallel_rank() + sequence_parallel_length = hidden_states.shape[0] // tp_size + hidden_states = hidden_states[ + tp_rank * sequence_parallel_length : (tp_rank + 1) * sequence_parallel_length + ] + return hidden_states.unsqueeze(1).contiguous().requires_grad_(True) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.skipif( + Utils.world_size % 8 != 0, reason="requires world size divisible by 8 for pp2/cp2/tp2/ep2/etp2" +) +@pytest.mark.internal +@pytest.mark.parametrize("dispatcher", ["alltoall", "deepep", "hybridep"]) +def test_sequence_packing_thd_e2e_proxy_model(dispatcher): + """Run packed THD attention + MoE forward/backward with major parallelisms enabled.""" + if not is_te_min_version("2.9.0"): + pytest.skip("SFT sequence packing requires Transformer Engine >= 2.9.0") + if dispatcher == "deepep" and not is_deep_ep_available(): + pytest.skip("Deep EP is not available") + if dispatcher == "hybridep" and not is_hybrid_ep_available(): + pytest.skip("Hybrid EP is not available") + + tp_size, pp_size, cp_size, ep_size, etp_size = 2, 2, 2, 2, 2 + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + context_parallel_size=cp_size, + expert_model_parallel_size=ep_size, + expert_tensor_parallel_size=etp_size, + ) + _set_random_seed(seed_=123, data_parallel_random_init=False) + + try: + spec = get_gpt_layer_with_transformer_engine_spec(num_experts=4, moe_grouped_gemm=False) + transformer_config = TransformerConfig( + num_layers=4, + hidden_size=1024, + ffn_hidden_size=2048, + moe_ffn_hidden_size=2048, + num_attention_heads=8, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + context_parallel_size=cp_size, + expert_model_parallel_size=ep_size, + expert_tensor_parallel_size=etp_size, + sequence_parallel=True, + sequence_packing_scheduler="dp_balanced", + max_seqlen_per_dp_cp_rank=1024, + cp_comm_type="p2p", + num_moe_experts=4, + moe_router_topk=2, + moe_router_load_balancing_type="aux_loss", + moe_token_dispatcher_type=( + "flex" if dispatcher in ("deepep", "hybridep") else dispatcher + ), + moe_flex_dispatcher_backend=( + dispatcher if dispatcher in ("deepep", "hybridep") else "deepep" + ), + moe_grouped_gemm=False, + moe_router_dtype="fp32", + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + autocast_dtype=torch.bfloat16, + bf16=True, + add_bias_linear=False, + attention_dropout=0.0, + hidden_dropout=0.0, + use_cpu_initialization=True, + ) + transformer_block = TransformerBlock(transformer_config, spec).cuda().to(torch.bfloat16) + + torch.manual_seed(1000 + torch.distributed.get_rank()) + seqlens = [257, 509, 1021] + hidden_states = _make_sharded_thd_hidden_states( + seqlens, transformer_config.hidden_size, cp_size, tp_size, torch.bfloat16 + ) + packed_seq_params = _make_thd_packed_seq_params(seqlens, cp_size, tp_size) + + output = transformer_block( + hidden_states=hidden_states, attention_mask=None, packed_seq_params=packed_seq_params + ) + assert output.shape == hidden_states.shape + assert torch.isfinite(output).all() + + loss = output.float().square().mean() + loss.backward() + + assert hidden_states.grad is not None + assert hidden_states.grad.shape == hidden_states.shape + assert torch.isfinite(hidden_states.grad).all() + assert any( + param.grad is not None and torch.isfinite(param.grad).all() + for param in transformer_block.parameters() + if param.requires_grad + ) + finally: + reset_hybrid_ep_buffer() + Utils.destroy_model_parallel() + + @pytest.mark.skipif( not is_deep_ep_available() and not is_hybrid_ep_available(), reason="Deep EP and Hybrid EP are not available", @@ -470,6 +643,7 @@ def setup_method(self, method): pass def teardown_method(self, method): + reset_hybrid_ep_buffer() Utils.destroy_model_parallel() @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") diff --git a/uv.lock b/uv.lock index 62ea93ec3ac..a2628601c01 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", @@ -305,7 +305,7 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ @@ -359,10 +359,10 @@ name = "anyio" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "idna" }, { name = "sniffio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } wheels = [ @@ -797,7 +797,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -984,7 +984,7 @@ name = "click" version = "8.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } wheels = [ @@ -1153,8 +1153,8 @@ name = "cryptography" version = "46.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a4/ba/04b1bd4218cbc58dc90ce967106d51582371b898690f3ae0402876cc4f34/cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759", size = 750542, upload-time = "2026-03-25T23:34:53.396Z" } wheels = [ @@ -1213,7 +1213,7 @@ name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1a/fe/7351d7e586a8b4c9f89731bfe4cf0148223e8f9903ff09571f78b3fb0682/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556", size = 5744254, upload-time = "2026-03-11T00:12:29.798Z" }, @@ -1266,37 +1266,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] curand = [ - { name = "nvidia-curand", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-curand", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or sys_platform == 'linux' or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] [[package]] @@ -1564,8 +1564,8 @@ name = "emerging-optimizers" version = "0.2.0" source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.2.0#1effa026ff096b7fa1063ca2fba19d98be6e6cdf" } dependencies = [ - { name = "absl-py", marker = "python_full_version >= '3.12'" }, - { name = "torch", marker = "python_full_version >= '3.12' and sys_platform == 'never'" }, + { name = "absl-py", marker = "python_full_version >= '3.12' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "torch", marker = "(python_full_version >= '3.12' and sys_platform == 'never') or (python_full_version < '3.12' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'never' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] [[package]] @@ -1573,7 +1573,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -2371,7 +2371,7 @@ dependencies = [ { name = "filelock" }, { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or sys_platform != 'win32' or extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, { name = "fsspec", version = "2026.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging" }, { name = "pyyaml" }, { name = "requests" }, @@ -2941,7 +2941,7 @@ resolution-markers = [ "python_full_version < '3.11' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version < '3.11'" }, + { name = "mdurl", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } wheels = [ @@ -3015,7 +3015,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version >= '3.11'" }, + { name = "mdurl", marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ @@ -3187,6 +3187,7 @@ dev = [ { name = "mamba-ssm" }, { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "multi-storage-client" }, + { name = "nvidia-cudnn-frontend" }, { name = "nvidia-modelopt", marker = "(sys_platform != 'darwin' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "nvidia-resiliency-ext" }, { name = "nvtx" }, @@ -3321,6 +3322,7 @@ requires-dist = [ { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy" }, + { name = "nvidia-cudnn-frontend", marker = "extra == 'dev'" }, { name = "nvidia-modelopt", extras = ["torch"], marker = "sys_platform != 'darwin' and extra == 'dev'" }, { name = "nvidia-resiliency-ext", marker = "extra == 'dev'", git = "https://github.com/NVIDIA/nvidia-resiliency-ext.git?rev=15a851565a4ce846c04431ecb0cf09903ab4837e" }, { name = "nvtx", marker = "extra == 'dev'", specifier = "~=0.2" }, @@ -3653,7 +3655,7 @@ name = "multidict" version = "6.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } wheels = [ @@ -4110,24 +4112,35 @@ name = "numpy" version = "2.0.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine != 's390x' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine == 's390x' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine != 's390x' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine == 's390x' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" } wheels = [ @@ -4280,7 +4293,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/74/f4c001f4714c3ad9ce037e18cf2b9c64871a84951eaa0baf683a9ca9301c/numpy-2.4.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119", size = 12509075, upload-time = "2026-03-29T13:21:57.644Z" }, ] - [[package]] name = "nvdlfw-inspect" version = "0.2.2" @@ -4338,7 +4350,7 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, @@ -4348,24 +4360,27 @@ wheels = [ [[package]] name = "nvidia-cudnn-frontend" -version = "1.22.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/7d/28ab9cb9119fc6a3a383d943448ab310fe787daf784869b167dc7269969f/nvidia_cudnn_frontend-1.22.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbd3100ae212dd1f4691f8c096fe3aded46491f9a6cb258bfb802d07ca1a88fc", size = 2670597, upload-time = "2026-04-03T02:27:56.886Z" }, - { url = "https://files.pythonhosted.org/packages/8b/b4/976996f1ab721bbcae4b7379652949ddcd41803817d4b65b9bd0d726aa60/nvidia_cudnn_frontend-1.22.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62bf9c8569caf4d9518dae0755507ad36a4e311726aa015fde104c38a1630f76", size = 2811815, upload-time = "2026-04-03T02:32:24.504Z" }, - { url = "https://files.pythonhosted.org/packages/2b/56/755412cf4ce5ad95bcb00be3144c8e1fa07cbbae073f31a7b75ddec96ca0/nvidia_cudnn_frontend-1.22.0-cp310-cp310-win_amd64.whl", hash = "sha256:22748b41049d02c029719467924ea20d928517dd8f35e204a390f97407298eb2", size = 2260435, upload-time = "2026-04-03T02:24:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ff/e4955b6fdff929ddf04a1252facae6201b308e001c91c690e96f65c4e90a/nvidia_cudnn_frontend-1.22.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdff54c945fbabf9da06fd64ded60cf1ec94d580474f5746786c0effd759fedc", size = 2672347, upload-time = "2026-04-03T02:28:51.106Z" }, - { url = "https://files.pythonhosted.org/packages/52/27/62fc6e2cddff7d6396be3685342ceec1c12fe2ee50e6f31d270887ecb5ad/nvidia_cudnn_frontend-1.22.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb50bd2758c6d47c6210451c5c1932ed16e7563d7629228f4cc97edc0e01d0c5", size = 2814387, upload-time = "2026-04-03T02:32:47.972Z" }, - { url = "https://files.pythonhosted.org/packages/d5/4f/de06583ec21313f31d8b83bc2164e88fc22f5b48d8eb5cb45490fcf7c262/nvidia_cudnn_frontend-1.22.0-cp311-cp311-win_amd64.whl", hash = "sha256:49f817377a19e10e4aafa5797cd68315739dfdb2fc6a67dd1052b64c805d24ec", size = 2261332, upload-time = "2026-04-03T02:25:17.241Z" }, - { url = "https://files.pythonhosted.org/packages/7e/f1/67681e585abd98f968298c771b72830ce984a90fd0d787098d2ea2ba55c7/nvidia_cudnn_frontend-1.22.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc9c12891d5427ef49b72b26df2b7889d623086d77c9e33b021c2de417d3e4dc", size = 2673215, upload-time = "2026-04-03T02:29:41.421Z" }, - { url = "https://files.pythonhosted.org/packages/0e/46/95b7779a2f71dfccce1783cc5ac210dda0124b93f8bf66cf62ed3d9ce0a5/nvidia_cudnn_frontend-1.22.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ffa05699d71795372f112fa2361c13be716fa3fda911c1e809903163ea5d11", size = 2815106, upload-time = "2026-04-03T02:33:11.473Z" }, - { url = "https://files.pythonhosted.org/packages/61/47/522e84a37eedb1f680e74df449d39fe6f8641779523313d1a8522d449766/nvidia_cudnn_frontend-1.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:81fde93d9b86ad631e17da1e2c103c4a7a541ec7abcb7f9a121cbd018c8eff26", size = 2262120, upload-time = "2026-04-03T02:25:40.18Z" }, - { url = "https://files.pythonhosted.org/packages/c7/93/43541b581207024824cb740f429bf882aaf3bde3633bd4099393dd9c0c16/nvidia_cudnn_frontend-1.22.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9bdf48cf989b2a77f8b52623fc31c078362fd34389207d11cdb0b5624a7b311", size = 2673259, upload-time = "2026-04-03T02:30:30.634Z" }, - { url = "https://files.pythonhosted.org/packages/9b/5b/af9da5a455064380e68a441b9cfa1f1212dd6363bd02b5aa696d319bd211/nvidia_cudnn_frontend-1.22.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d02c4b4aae3e243ddb08ad4eb939988bcf7b1aefe25f5d400f6858c7276a6631", size = 2815032, upload-time = "2026-04-03T02:33:34.171Z" }, - { url = "https://files.pythonhosted.org/packages/ba/1d/3a15b719817ca6241e5f3a7a38608af21a3259e550a5dee5520e29adac00/nvidia_cudnn_frontend-1.22.0-cp313-cp313-win_amd64.whl", hash = "sha256:4906a38954725e35bc8431874f4d9db60d50e0d9dbc40ecaf8e5f40df545350b", size = 2262156, upload-time = "2026-04-03T02:26:03.322Z" }, - { url = "https://files.pythonhosted.org/packages/27/ec/8c9b53a9174cca2d0062cbd8cb7c31403a38cb4c79984a9c554830cac5e9/nvidia_cudnn_frontend-1.22.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f650058bda46a6542dfc3d021803021e7932e1cd6bb78cf46e81fa219717b5e", size = 2674887, upload-time = "2026-04-03T02:31:21.166Z" }, - { url = "https://files.pythonhosted.org/packages/89/bd/3464d181ec2d94085cab98fd5ea4d312478aa6cb16ff38994a9188ac9f05/nvidia_cudnn_frontend-1.22.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f30b0d6563d050ca1972efa594a31d5affe5c3eeb467542e715d7ee73e3b5b", size = 2815841, upload-time = "2026-04-03T02:33:56.66Z" }, - { url = "https://files.pythonhosted.org/packages/a5/fd/bdec32a32b44f52b60a03f43e8619552ea0eb90a61de06632a054bf17d6a/nvidia_cudnn_frontend-1.22.0-cp314-cp314-win_amd64.whl", hash = "sha256:5994400a7f76a1be5e327a9ac1a4a635ee734d2ac8a5875e52481c52cf2b0922", size = 2263464, upload-time = "2026-04-03T02:26:26.553Z" }, +version = "1.24.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/05/bf31134b6e5d41c5a1e4abc65b5bb5461a73be74013da273b18fef7a2244/nvidia_cudnn_frontend-1.24.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8833079f0283948cb5f99a2dc0c8fbff29d320e6a5635c4f77fa7eaf877043b", size = 3222069, upload-time = "2026-05-20T05:01:12.464Z" }, + { url = "https://files.pythonhosted.org/packages/50/4d/861c75757e688a5094da621871688d7583b52c7cb3ad75d0d5ab1dcdff68/nvidia_cudnn_frontend-1.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5736397ab8f29e06731960e055d27f65e8fda70c70f988e8bd91b671a3506999", size = 3370408, upload-time = "2026-05-20T05:01:36.098Z" }, + { url = "https://files.pythonhosted.org/packages/de/13/cee7f47acbb1d85a2019522f88238cbd4ac182a65fdea5d41e504b7c48be/nvidia_cudnn_frontend-1.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:77bc9f3203c677f74b6cedf84125514b4881dc82f4177cc4ab33949693abe6aa", size = 2764295, upload-time = "2026-05-20T05:01:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/81/75/767973a56b98d2a8fbc04c78fc28684cdb0df7c032ba4858090243df81cc/nvidia_cudnn_frontend-1.24.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c0f39f211bb105798c7a8617b1d674e01dbd538b97714025628ea7127bfaf8a", size = 3223392, upload-time = "2026-05-20T05:02:56.092Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8c/f5226ab5163dfffbe82ff41b9e1fbf649c908c077a5416bb16a1c3634cfa/nvidia_cudnn_frontend-1.24.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dec07dcfea168792098b9a2652ec465d79e228d3d17a2f86a08404b487336530", size = 3372046, upload-time = "2026-05-20T05:03:20.091Z" }, + { url = "https://files.pythonhosted.org/packages/a7/bf/fa08a8bd953ad8db7ea3f64676b30f1cb2846f6518595a6faf7fb3558db4/nvidia_cudnn_frontend-1.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:b461259b85b7a7e3a1c41b02c33ce4fde0dbcde7e0a227a968dedb74d311e2c3", size = 2764877, upload-time = "2026-05-20T05:04:21.965Z" }, + { url = "https://files.pythonhosted.org/packages/25/f4/a57efcbeb1dec0a047fa8f8aaa4defa4935db7d52e8afaf83224f1258ec6/nvidia_cudnn_frontend-1.24.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b4398cecbaa555baa73a9b8716233632e3c16259c6ab999d83c51ca3b8fd09c", size = 3224420, upload-time = "2026-05-20T05:04:50.051Z" }, + { url = "https://files.pythonhosted.org/packages/19/b7/c1b8de8292b8fe28b2ffc95601a0b69392536a9515315bcfcdb0b07d2af8/nvidia_cudnn_frontend-1.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:144bfe09098d681d4c793c867fff53b1fb7c49f845324d5d0c52d824b14b74f5", size = 3376971, upload-time = "2026-05-20T05:05:19.998Z" }, + { url = "https://files.pythonhosted.org/packages/dc/00/284b8de99fbc1e8fd91c292a024943ee61d3361aa669c435ee44e14b6498/nvidia_cudnn_frontend-1.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:5476d6a51ebaf5ef04e462e0052f1d9bca1af6274f738cb509715b4cf443a8df", size = 2765023, upload-time = "2026-05-20T05:05:40.269Z" }, + { url = "https://files.pythonhosted.org/packages/41/3e/430941e91a0c5234c79aa0bd6dcdb27c0e3a443f66953f881f8cb3428e93/nvidia_cudnn_frontend-1.24.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccf1d2352f4b82fafbfb803512493c7a211224b424b6a78fbf45e42a94dcdb81", size = 3224661, upload-time = "2026-05-20T05:06:01.577Z" }, + { url = "https://files.pythonhosted.org/packages/d8/95/02691a6f0db4c2194899579a9c196df8990b140e0503fc6be1f09e7aa063/nvidia_cudnn_frontend-1.24.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec45b08e0ab511f61532bc980c343c15eb5eda9ff14a3c80e75bc0aa3776860c", size = 3376128, upload-time = "2026-05-20T05:06:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c5/e6c9ef439e167675e64fd1ca025df79211cafdf70ec9edbf6b99f876cafd/nvidia_cudnn_frontend-1.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:2d02744a46726d262d80ed54299fae6491e4385e7d580eb6a027fb5b3b2c1db8", size = 2765241, upload-time = "2026-05-20T05:06:45.6Z" }, + { url = "https://files.pythonhosted.org/packages/f0/fc/c5a1d386f22dc8e17304874368ba213ff8c093093e9b0ccc3b2bd81a1d12/nvidia_cudnn_frontend-1.24.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:190857577a11d22b62da1863cb1a4b72692f98913d1c9c2e6a72224d08685575", size = 3226124, upload-time = "2026-05-20T05:07:14.426Z" }, + { url = "https://files.pythonhosted.org/packages/2e/39/e7f12c1a640174bdebbbf87be19819483c9da9f8f3af948f5dcb46cf501d/nvidia_cudnn_frontend-1.24.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7e9aca4b6ce4d4bd484f01a1c4f530a9bea317cd69d21760d582b47e05514d8", size = 3375383, upload-time = "2026-05-20T05:07:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/16/f8/cae1cffd3ab944301e3a736395f85925a887fc5255c100f561638d6b2fba/nvidia_cudnn_frontend-1.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8968eb9dd9a71fe3d64b55d1e9731cffb7272578a9a39c9bf816f5e27c3f14e", size = 2765850, upload-time = "2026-05-20T05:07:59.264Z" }, + { url = "https://files.pythonhosted.org/packages/9e/27/c04b542fd2a882fabed1cb4e6778a8a6159f1123e00c2f0da8383690c713/nvidia_cudnn_frontend-1.24.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65044724acc5fcb92ab829c2d2fcefc7fd4419881030e04358a66a4bcbce5d43", size = 3230149, upload-time = "2026-05-20T05:08:21.883Z" }, + { url = "https://files.pythonhosted.org/packages/fc/17/89a0eccbf5be9455c395f977022e299c910f1d7d5b0820661908f552ac70/nvidia_cudnn_frontend-1.24.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37c6ef88c7cbc41eab6e36d5c6715ff0d6c639f76ca12c65ce7a24b453e184eb", size = 3382455, upload-time = "2026-05-20T05:08:48.412Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d2/dac91d6a6fa2e6c07b59239f01fb07472f36d8368c29840647cfcd1c7dd0/nvidia_cudnn_frontend-1.24.0-cp314-cp314t-win_amd64.whl", hash = "sha256:79e902e124123d84d52fa06c0931415cccfa71ff8550c1c156dd3539bffeda5c", size = 2791017, upload-time = "2026-05-20T05:09:07.477Z" }, ] [[package]] @@ -4373,7 +4388,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -4405,9 +4420,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -4420,7 +4435,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -4715,11 +4730,11 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "protobuf" }, - { name = "typing-extensions" }, + { name = "protobuf", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/93/942d2a0f6a70538eea042ce0445c8aefd46559ad153469986f29a743c01c/onnx-1.21.0.tar.gz", hash = "sha256:4d8b67d0aaec5864c87633188b91cc520877477ec0254eda122bef8be43cd764", size = 12074608, upload-time = "2026-03-27T21:33:36.118Z" } wheels = [ @@ -4814,12 +4829,12 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" } }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "sympy", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b2/a5/acc43c8fa6edbc584d127fb6bbd13ae9ebfc01b9675c74e0da2de15fa4a6/onnx_ir-0.2.0.tar.gz", hash = "sha256:8bad3906691987290789b26d05e0dbff467029a0b1e411e12e4cae02e43503e4", size = 141693, upload-time = "2026-02-24T02:31:10.998Z" } wheels = [ @@ -4890,13 +4905,13 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" } }, - { name = "onnx-ir", version = "0.2.0", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "typing-extensions" }, + { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnx-ir", version = "0.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "packaging", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/538fdeb0e25bed5d7e0f954af5710543e2629499fb74381afc3333f8a8ae/onnxscript-0.6.2.tar.gz", hash = "sha256:abb2e6f464db40c9b8c7fbb3e64cca04cf3f4495e67c4eda5eac17b784191ce3", size = 590865, upload-time = "2026-02-10T22:53:39.638Z" } wheels = [ @@ -4994,7 +5009,7 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.41.0" +version = "1.42.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -5012,12 +5027,11 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "importlib-metadata", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "typing-extensions", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/47/8e/3778a7e87801d994869a9396b9fc2a289e5f9be91ff54a27d41eace494b0/opentelemetry_api-1.41.0.tar.gz", hash = "sha256:9421d911326ec12dee8bc933f7839090cad7a3f13fcfb0f9e82f8174dc003c09", size = 71416, upload-time = "2026-04-09T14:38:34.544Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/ca/25288069c399be6769159d9fb7b1190b603537d82aad2fa2746a0cc2c8c6/opentelemetry_api-1.42.0.tar.gz", hash = "sha256:ea84c893ad177791d138e0349d6ceebd8d3bf006440900400ce220008dafc372", size = 72300, upload-time = "2026-05-19T09:46:29.885Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/ee/99ab786653b3bda9c37ade7e24a7b607a1b1f696063172768417539d876d/opentelemetry_api-1.41.0-py3-none-any.whl", hash = "sha256:0e77c806e6a89c9e4f8d372034622f3e1418a11bdbe1c80a50b3d3397ad0fa4f", size = 69007, upload-time = "2026-04-09T14:38:11.833Z" }, + { url = "https://files.pythonhosted.org/packages/1b/0b/be5daf659b82b525338fde371dfcfab09b606a19bb5620c37076964710ec/opentelemetry_api-1.42.0-py3-none-any.whl", hash = "sha256:558d88f88192a973579910ef6f2c13db47a268d5ec2e53e83e50e74a39a02922", size = 61310, upload-time = "2026-05-19T09:46:06.561Z" }, ] [[package]] @@ -5064,7 +5078,7 @@ wheels = [ [[package]] name = "opentelemetry-exporter-prometheus" -version = "0.62b0" +version = "0.63b0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -5082,13 +5096,13 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "opentelemetry-api", version = "1.41.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, - { name = "opentelemetry-sdk", version = "1.41.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-api", version = "1.42.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-sdk", version = "1.42.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "prometheus-client", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0f/ec/fa8a722199dc2e75dc582779d62207b00b0bdb014b5635594afa0cf3ee43/opentelemetry_exporter_prometheus-0.62b0.tar.gz", hash = "sha256:4d1106566a9b3e8dff028e69e9f2dc90723e6b431c900ff8c72982fcf11dbae5", size = 15441, upload-time = "2026-04-09T14:38:38.934Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/2c/0643113a5bef20e8242f7ae7915913fab61e8c901d391518a0aefa2da6fc/opentelemetry_exporter_prometheus-0.63b0.tar.gz", hash = "sha256:76b52078ee70131542e53d5cf1942cadd6d5628e7a1bf1f60047f29fa079e9b1", size = 15231, upload-time = "2026-05-19T09:46:35.1Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/1e/43645fadd561471af2aec95906a3dd54af1a8e7782322310e802a810ad3a/opentelemetry_exporter_prometheus-0.62b0-py3-none-any.whl", hash = "sha256:cd7e8acae3be5f425ffa2e0864eea474fa7a40706f786de7a2d23846573d8f75", size = 13278, upload-time = "2026-04-09T14:38:19.367Z" }, + { url = "https://files.pythonhosted.org/packages/2b/48/18e2b0eec9242beb168b1100ed6c602c2a378d0ccb779d1c0a1b85b9ba89/opentelemetry_exporter_prometheus-0.63b0-py3-none-any.whl", hash = "sha256:0cfe4846bf5905f096a4d9678ffe25c7fe6f662f6c7282d2b191138d6bf487fb", size = 12466, upload-time = "2026-05-19T09:46:15.025Z" }, ] [[package]] @@ -5147,7 +5161,7 @@ wheels = [ [[package]] name = "opentelemetry-sdk" -version = "1.41.0" +version = "1.42.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -5165,13 +5179,13 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "opentelemetry-api", version = "1.41.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, - { name = "opentelemetry-semantic-conventions", version = "0.62b0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-api", version = "1.42.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-semantic-conventions", version = "0.63b0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "typing-extensions", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f8/0e/a586df1186f9f56b5a0879d52653effc40357b8e88fc50fe300038c3c08b/opentelemetry_sdk-1.41.0.tar.gz", hash = "sha256:7bddf3961131b318fc2d158947971a8e37e38b1cd23470cfb72b624e7cc108bd", size = 230181, upload-time = "2026-04-09T14:38:47.225Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/c9/dabaaf1c754a57b82b5a36aeca3806d92c1877ccfb12a697b65f88bf027c/opentelemetry_sdk-1.42.0.tar.gz", hash = "sha256:2479e462cc69357825c2c847ce4a601bc1b17e1279aa7f80d3490f0ae614d0e5", size = 239072, upload-time = "2026-05-19T09:46:42.992Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/13/a7825118208cb32e6a4edcd0a99f925cbef81e77b3b0aedfd9125583c543/opentelemetry_sdk-1.41.0-py3-none-any.whl", hash = "sha256:a596f5687964a3e0d7f8edfdcf5b79cbca9c93c7025ebf5fb00f398a9443b0bd", size = 180214, upload-time = "2026-04-09T14:38:30.657Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7d/16bf9a9d42ebbd1679e0cda018d57a0712f3b6f6f1e7ae5ef3c7ee5927c0/opentelemetry_sdk-1.42.0-py3-none-any.whl", hash = "sha256:ec4a4f69e15220b3d7bccd93217aac745682bb6435b9381f7bb44cb7e07b4f2b", size = 170879, upload-time = "2026-05-19T09:46:25.871Z" }, ] [[package]] @@ -5217,7 +5231,7 @@ wheels = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.62b0" +version = "0.63b0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -5235,12 +5249,12 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "opentelemetry-api", version = "1.41.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-api", version = "1.42.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "typing-extensions", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/b0/c14f723e86c049b7bf8ff431160d982519b97a7be2857ed2247377397a24/opentelemetry_semantic_conventions-0.62b0.tar.gz", hash = "sha256:cbfb3c8fc259575cf68a6e1b94083cc35adc4a6b06e8cf431efa0d62606c0097", size = 145753, upload-time = "2026-04-09T14:38:48.274Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/f8/be4625838aae098c2f9fbdc062a1b3128ebb9e799b891b654ee8cad94897/opentelemetry_semantic_conventions-0.63b0.tar.gz", hash = "sha256:cfea295264654fa324fcef24aa56fb1836fdc0da27db128645dc6aa76115cc6c", size = 148333, upload-time = "2026-05-19T09:46:44.01Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/6c/5e86fa1759a525ef91c2d8b79d668574760ff3f900d114297765eb8786cb/opentelemetry_semantic_conventions-0.62b0-py3-none-any.whl", hash = "sha256:0ddac1ce59eaf1a827d9987ab60d9315fb27aea23304144242d1fcad9e16b489", size = 231619, upload-time = "2026-04-09T14:38:32.394Z" }, + { url = "https://files.pythonhosted.org/packages/8f/6f/8d0ce225b8fdbb72c97cf4130107d861eafcb3d8e5c3f5891e8556177316/opentelemetry_semantic_conventions-0.63b0-py3-none-any.whl", hash = "sha256:1f3962732b04f43e4fef28173c9a3615b8847b4b2d6386fdc085361b29875ab9", size = 203712, upload-time = "2026-05-19T09:46:27.569Z" }, ] [[package]] @@ -5328,27 +5342,7 @@ wheels = [ name = "packaging" version = "26.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] @@ -5358,30 +5352,41 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine != 's390x' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine == 's390x' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine != 's390x' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and platform_machine == 's390x' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or extra == 'extra-13-megatron-core-dev'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11' or extra == 'extra-13-megatron-core-dev'" }, + { name = "pytz", marker = "python_full_version < '3.11' or extra == 'extra-13-megatron-core-dev'" }, + { name = "tzdata", marker = "python_full_version < '3.11' or extra == 'extra-13-megatron-core-dev'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -6241,7 +6246,7 @@ name = "pyjwt" version = "2.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" } wheels = [ @@ -6624,10 +6629,10 @@ default = [ { name = "grpcio" }, { name = "opencensus" }, { name = "opentelemetry-exporter-prometheus", version = "0.54b1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, - { name = "opentelemetry-exporter-prometheus", version = "0.62b0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-exporter-prometheus", version = "0.63b0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk", version = "1.33.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, - { name = "opentelemetry-sdk", version = "1.41.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-sdk", version = "1.42.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "prometheus-client" }, { name = "py-spy" }, { name = "pydantic" }, @@ -6643,7 +6648,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -7859,7 +7864,7 @@ version = "0.52.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/68/79977123bb7be889ad680d79a40f339082c1978b5cfcf62c2d8d196873ac/starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933", size = 2653702, upload-time = "2026-01-18T13:34:11.062Z" } wheels = [ @@ -7871,7 +7876,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version < '3.13' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -7963,7 +7968,7 @@ resolution-markers = [ dependencies = [ { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3c/b9/ea25aba62c688a87d7d7d9cc5926d602e2f9e84fa72586825486fb180b7e/tensorstore-0.1.74.tar.gz", hash = "sha256:a062875f27283d30ce4959c408c253ecb336fce8e3f9837c064e3d30cda79203", size = 6795605, upload-time = "2025-04-24T15:42:18.829Z" } wheels = [ @@ -8020,7 +8025,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (python_full_version >= '3.11' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] @@ -8258,21 +8263,21 @@ name = "torch" version = "2.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-bindings", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "filelock", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "jinja2", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "filelock", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "jinja2", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "sympy", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "triton", marker = "sys_platform == 'never'" }, - { name = "typing-extensions", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "setuptools", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "sympy", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "triton", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ac/f2/c1690994afe461aae2d0cac62251e6802a703dec0a6c549c02ecd0de92a9/torch-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c0d7fcfbc0c4e8bb5ebc3907cbc0c6a0da1b8f82b1fc6e14e914fa0b9baf74e", size = 80526521, upload-time = "2026-03-23T18:12:06.86Z" }, @@ -8331,7 +8336,7 @@ name = "tqdm" version = "4.67.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } wheels = [ @@ -8510,7 +8515,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/62/f2/368268300fb8af33743508d738ef7bb4d56afdb46c6d9c0fa3dd515df171/uvicorn-0.43.0.tar.gz", hash = "sha256:ab1652d2fb23abf124f36ccc399828558880def222c3cb3d98d24021520dc6e8", size = 85686, upload-time = "2026-04-03T18:37:48.984Z" } wheels = [