diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml index b9e97b18946..7013df60dc2 100644 --- a/.github/copy-pr-bot.yaml +++ b/.github/copy-pr-bot.yaml @@ -1,4 +1,4 @@ enabled: true auto_sync_draft: false auto_sync_ready: true -trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "gautham-kollu", "hxbai", "jaredcasper", "jkamalu", "jon-barker", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mkhona-nvidia", "pablo-garay", "pthombre", "rogerwaleffe", "sanandaraj5597", "santhnm2", "sbak5", "shanmugamr1992", "sidsingh-nvidia", "skyw", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"] +trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "gautham-kollu", "hxbai", "jaredcasper", "jiemingz", "jkamalu", "jon-barker", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mehraakash", "mkhona-nvidia", "pablo-garay", "parthmannan", "pthombre", "rogerwaleffe", "sanandaraj5597", "santhnm2", "sbak5", "shanmugamr1992", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"] diff --git a/.github/workflows/auto-update-copy-pr-bot.yml b/.github/workflows/auto-update-copy-pr-bot.yml index 25b3a3d2a30..969c46e3fdd 100644 --- a/.github/workflows/auto-update-copy-pr-bot.yml +++ b/.github/workflows/auto-update-copy-pr-bot.yml @@ -9,6 +9,7 @@ jobs: auto-update-copy-pr-bot: runs-on: ubuntu-latest environment: nemo-ci + if: github.repository == 'NVIDIA/Megatron-LM' steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index ca6e59a1625..4a4a1a2cad1 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -233,11 +233,26 @@ jobs: with: python-version: 3.12 + - name: Install GH CLI + shell: bash -x -e -u -o pipefail {0} + run: | + apt-get update + apt-get install -y gh + - name: Get PR info id: get-pr-info if: startsWith(github.ref, 'refs/heads/pull-request/') uses: nv-gha-runners/get-pr-info@main + - name: Has lts label + id: has-lts-label + env: + GH_TOKEN: ${{ secrets.PAT }} + run: | + PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} + HAS_LTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "container::lts")') || echo "false" + echo "main=$HAS_LTS_LABEL" | tee -a $GITHUB_OUTPUT + - name: Download test data shell: bash env: @@ -276,6 +291,22 @@ jobs: echo "$LAST_PRS" | tee -a $GITHUB_OUTPUT echo "EOF" | tee -a $GITHUB_OUTPUT + - name: Parse baseimage + shell: bash + id: base-image + env: + HAS_LTS_LABEL: ${{ steps.has-lts-label.outputs.main }} + run: | + if [ "$HAS_LTS_LABEL" == "true" ]; then + NGC_VERSION=$(cat docker/.ngc_version.lts) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=lts" | tee -a $GITHUB_OUTPUT + else + NGC_VERSION=$(cat docker/.ngc_version.dev) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=dev" | tee -a $GITHUB_OUTPUT + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -287,7 +318,8 @@ jobs: context: . target: main build-args: | - FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:25.09-py3 + FROM_IMAGE_NAME=${{ steps.base-image.outputs.version }} + IMAGE_TYPE=${{ steps.base-image.outputs.image_type }} cache-from: | type=registry,ref=${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}-buildcache,mode=max type=registry,ref=${{ env.container-registry }}/megatron-lm:main-buildcache,mode=max diff --git a/.github/workflows/multi-approval-bot.yml b/.github/workflows/multi-approval-bot.yml deleted file mode 100644 index e8507605aa7..00000000000 --- a/.github/workflows/multi-approval-bot.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Codeowners Approval Workflow" - -on: - push: - branches: - - "pull-request/[0-9]+" - merge_group: - types: [checks_requested] - -jobs: - pre-flight: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5 - if: github.repository == 'NVIDIA/Megatron-LM' - - codeowners-approval: - needs: [pre-flight] - runs-on: ubuntu-latest - environment: nemo-ci - if: | - !(needs.pre-flight.outputs.docs_only == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - || needs.pre-flight.outputs.is_deployment_workflow == 'true') - steps: - - name: Get PR info - id: get-pr-info - if: startsWith(github.ref, 'refs/heads/pull-request/') - uses: nv-gha-runners/get-pr-info@main - - - name: Checkout action - uses: actions/checkout@v3 - with: - repository: noamelf/codeowner-multi-approval-action - ref: v0.1 - path: codeowner-multi-approval-action - - - name: Check Codeowners Approval - uses: ./codeowner-multi-approval-action - with: - pr-number: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} - repo-name: ${{ github.repository }} - github-token: ${{ secrets.PAT }} - - multi-approval-bot-summary: - needs: [pre-flight, codeowners-approval] - if: | - ( - needs.pre-flight.outputs.docs_only == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - || needs.pre-flight.outputs.is_deployment_workflow == 'true' - || always() - ) - && github.repository == 'NVIDIA/Megatron-LM' - && !cancelled() - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Result - env: - GH_TOKEN: ${{ github.token }} - GITHUB_RUN_ID: ${{ github.run_id }} - SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }} - run: | - FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0 - - if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then - echo "✅ All previous jobs completed successfully" - exit 0 - else - echo "❌ Found $FAILED_JOBS failed job(s)" - # Show which jobs failed - gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name' - exit 1 - fi diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index 0658daaa9ec..d67225311f6 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -53,10 +53,12 @@ test:build_image: parallel: matrix: - IMAGE: CI_MCORE_LTS_IMAGE - FILE: Dockerfile.ci.lts - BASE_IMAGE: nvcr.io/nvidia/pytorch:24.01-py3 + FILE: Dockerfile.ci.dev + IMAGE_TYPE: lts + BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev + IMAGE_TYPE: dev BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index 11c8403e584..00000000000 --- a/CODEOWNERS +++ /dev/null @@ -1,26 +0,0 @@ -# Core -[Core] @mcore-reviewers/dev-core -megatron/core/ - -[CI][1] @mcore-reviewers/ci -.gitlab/ -.github/ -.gitlab-ci.yml -docker/ -tests/unit_tests/run_ci_test.sh -tests/test_utils/python_scripts/ -tests/functional_tests/python_test_utils/ -tests/functional_tests/shell_test_utils/ -megatron/core/transformer/transformer_block.py -megatron/core/transformer/transformer_layer.py - -^[Tests][1] @mcore-reviewers/ci -tests/functional_tests/test_cases/ -tests/functional_tests/recipes/ -tests/unit_tests/ - -[RL] @mcore-reviewers/rl -megatron/rl/ -examples/rl/ -tests/unit_tests/test_rl_utils.py -train_rl.py diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.dev @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/.ngc_version.lts b/docker/.ngc_version.lts new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.lts @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index 9ebda435f99..95fecdb3f9b 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -1,8 +1,6 @@ # syntax=docker/dockerfile:1.3-labs ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - FROM ${FROM_IMAGE_NAME} as main ENV PIP_CONSTRAINT="" ENV DEBIAN_FRONTEND=noninteractive @@ -25,16 +23,16 @@ RUN bash -ex <<"EOF" curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh EOF -ARG WHEEL_DIR COPY README.md pyproject.toml uv.lock /workspace/ COPY megatron/core/__init__.py /workspace/megatron/core/ COPY megatron/core/package_info.py /workspace/megatron/core/ +ARG IMAGE_TYPE=dev RUN --mount=type=cache,target=/root/.cache/uv \ bash -ex <<"EOF" export NVTE_CUDA_ARCHS="80;90;100" uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages uv sync --only-group build - uv sync --extra dev --extra mlm --link-mode copy --locked \ + uv sync --extra ${IMAGE_TYPE} --extra mlm --link-mode copy --locked \ --no-install-package torch \ --no-install-package torchvision \ --no-install-package triton \ diff --git a/docker/Dockerfile.ci.lts b/docker/Dockerfile.ci.lts deleted file mode 100644 index 7da27a03f1d..00000000000 --- a/docker/Dockerfile.ci.lts +++ /dev/null @@ -1,98 +0,0 @@ -# syntax=docker/dockerfile:1.3-labs - -ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - -FROM $FROM_IMAGE_NAME as build_mamba -WORKDIR /opt -ARG WHEEL_DIR -RUN MAMBA_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/state-spaces/mamba.git@v2.0.3 -w $WHEEL_DIR - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as build_causalconv1d -WORKDIR /opt -ARG WHEEL_DIR -RUN CAUSAL_CONV1D_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/Dao-AILab/causal-conv1d.git@v1.2.2.post1 -w $WHEEL_DIR - -FROM $FROM_IMAGE_NAME as build_groupedgemm -WORKDIR /opt -ARG WHEEL_DIR -RUN pip3 wheel -v git+https://github.com/fanshiqing/grouped_gemm@v1.1.2 -w $WHEEL_DIR - - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as main -ENV DEBIAN_FRONTEND=noninteractive - -RUN bash -ex <<"EOF" - apt-get update - apt-get install -y --no-install-recommends gettext python3-venv psmisc - apt-get clean - python -m venv /opt/jet - wget https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -O /usr/local/bin/yq - chmod a+x /usr/local/bin/yq -EOF - -ARG UV_VERSION=0.7.2 -ENV PATH="/root/.local/bin:$PATH" -RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh -ENV UV_PROJECT_ENVIRONMENT=/opt/venv -ENV PATH="$UV_PROJECT_ENVIRONMENT/bin:$PATH" -ENV VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT -ENV UV_LINK_MODE=copy - -RUN -ARG WHEEL_DIR -COPY README.md pyproject.toml uv.lock /workspace/ -COPY megatron/core/__init__.py /workspace/megatron/core/ -COPY megatron/core/package_info.py /workspace/megatron/core/ -COPY docker/common/ /workspace/docker/common/ -COPY --from=build_mamba $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_causalconv1d $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_groupedgemm $WHEEL_DIR/*.whl $WHEEL_DIR/ -RUN bash -ex <<"EOF" - uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages - - uv sync --extra lts --extra mlm --link-mode copy --locked \ - --no-install-package torch \ - --no-install-package torchvision \ - --no-install-package triton \ - --no-install-package nvidia-cublas-cu12 \ - --no-install-package nvidia-cuda-cupti-cu12 \ - --no-install-package nvidia-cuda-nvrtc-cu12 \ - --no-install-package nvidia-cuda-runtime-cu12 \ - --no-install-package nvidia-cudnn-cu12 \ - --no-install-package nvidia-cufft-cu12 \ - --no-install-package nvidia-cufile-cu12 \ - --no-install-package nvidia-curand-cu12 \ - --no-install-package nvidia-cusolver-cu12 \ - --no-install-package nvidia-cusparse-cu12 \ - --no-install-package nvidia-cusparselt-cu12 \ - --no-install-package nvidia-nccl-cu12 - - bash docker/common/install_source_wheels.sh --input-wheel-dir $WHEEL_DIR/ --environment lts -EOF -ENV PYTHONPATH="/opt/megatron-lm:$PYTHONPATH" -COPY assets/ /opt/data/ -ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python - -##### For NVIDIANS only ##### -FROM main as jet -ARG JET_API_VERSION -ENV PATH="$PATH:/opt/jet/bin" -RUN --mount=type=secret,id=JET_INDEX_URLS bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - python -m venv /opt/jet - /opt/jet/bin/pip install --no-cache-dir $JET_INDEX_URLS \ - jet-api==$JET_API_VERSION -EOF - -RUN --mount=type=secret,id=JET_INDEX_URLS \ - --mount=type=secret,id=LOGGER_INDEX_URL bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - LOGGER_INDEX_URL=$(cat /run/secrets/LOGGER_INDEX_URL) - uv pip install --no-cache-dir --upgrade $LOGGER_INDEX_URL "one-logger" - uv pip install --no-cache-dir --upgrade "setuptools<80.0.0" - uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=3.0" -EOF -### \ No newline at end of file diff --git a/examples/inference/gpt/gpt_dynamic_inference.py b/examples/inference/gpt/gpt_dynamic_inference.py index e5344fbb8be..251aa100cba 100644 --- a/examples/inference/gpt/gpt_dynamic_inference.py +++ b/examples/inference/gpt/gpt_dynamic_inference.py @@ -11,7 +11,7 @@ from collections import defaultdict from functools import partial from tqdm import tqdm -from typing import Dict, List, Optional +from typing import Dict, List, Tuple, Optional import torch from tqdm import tqdm @@ -28,18 +28,21 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.ssm.mamba_hybrid_layer_allocation import Symbols from megatron.core.tokenizers.text.utils.build_tokenizer import build_tokenizer from megatron.core.transformer.module import MegatronModule +from megatron.core.utils import get_attr_wrapped_model sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) ) from megatron.training import get_args, get_model as _get_model, get_tokenizer, initialize_megatron from megatron.training.checkpointing import load_checkpoint - -from megatron.core.utils import configure_nvtx_profiling from model_provider import model_provider from gpt_builders import gpt_builder +from mamba_builders import mamba_builder + +from megatron.core.utils import configure_nvtx_profiling import json @@ -54,17 +57,11 @@ from megatron.training import get_model as _get_model from megatron.training import get_tokenizer, initialize_megatron from megatron.training.checkpointing import load_checkpoint -from pretrain_gpt import model_provider import torch import io import megatron -torch.serialization.add_safe_globals([io.BytesIO]) -torch.serialization.add_safe_globals([megatron.core.rerun_state_machine.RerunState]) -torch.serialization.add_safe_globals([megatron.core.rerun_state_machine.RerunDiagnostic]) - - def add_dynamic_inference_args(parser: ArgumentParser) -> ArgumentParser: """Dynamic inference arguments.""" @@ -91,9 +88,16 @@ def get_model() -> MegatronModule: args = get_args() + if args.model_provider == "gpt": + model_builder = gpt_builder + elif args.model_provider == "mamba": + model_builder = mamba_builder + else: + raise ValueError(f"Invalid model provider {args.model_provider}") + # Build model. model = _get_model( - partial(model_provider, gpt_builder), + partial(model_provider, model_builder), wrap_with_ddp=False ) @@ -120,7 +124,10 @@ def get_model() -> MegatronModule: def get_inference_context( requests: List[Request], sampling_params: Optional[SamplingParams] = None, - calculate_max_sequence_length_from_requests: bool = True + calculate_max_sequence_length_from_requests: bool = True, + layer_type_list: Optional[List[str]] = None, + mamba_conv_states_shape: Optional[Tuple[int]] = None, + mamba_ssm_states_shape: Optional[Tuple[int]] = None, ): """The inference context manages the KV cache and other inference state.""" @@ -159,6 +166,9 @@ def get_inference_context( max_tokens_override=args.inference_dynamic_batching_max_tokens_override, tensor_model_parallel_size=args.tensor_model_parallel_size, materialize_only_last_token_logits=not args.return_log_probs, + layer_type_list=layer_type_list, + mamba_conv_states_shape=mamba_conv_states_shape, + mamba_ssm_states_shape=mamba_ssm_states_shape, cache_mla_latent=args.multi_latent_attention and args.cache_mla_latents, kv_lora_rank=args.kv_lora_rank if args.multi_latent_attention else None, qk_pos_emb_head_dim=args.qk_pos_emb_head_dim, @@ -369,21 +379,38 @@ def main(): termination_id=args.termination_id if args.termination_id is not None else tokenizer.eod, ) - # Requests, context, conroller. model = get_model() + + # Layer type list for hybrid models + decoder = get_attr_wrapped_model(model, "decoder") + layer_type_list = getattr(decoder, "layer_type_list", None) + if layer_type_list is not None and Symbols.MAMBA in layer_type_list: + (mamba_conv_states_shape, mamba_ssm_states_shape) = decoder.mamba_state_shapes_per_request() + else: + mamba_conv_states_shape = None + mamba_ssm_states_shape = None + + # Requests, context, controller. requests = build_requests(args, tokenizer, sampling_params) - context = get_inference_context(requests, sampling_params) + context = get_inference_context( + requests, + sampling_params, + layer_type_list=layer_type_list, + mamba_conv_states_shape=mamba_conv_states_shape, + mamba_ssm_states_shape=mamba_ssm_states_shape, + ) controller = get_inference_controller(model, context) # Validate all context_length's <= max_tokens. - invalid_prompt_length_map = {} - for request_idx, request in enumerate(requests): - if len(request.prompt_tokens) > context.max_tokens: - invalid_prompt_length_map[request_idx] = len(request.prompt_tokens) - assert not invalid_prompt_length_map, ( - "request idxs with prompts longer than context.max_tokens: " - ", ".join(f"{k}({v})" for k, v in invalid_prompt_length_map.items()) - ) + if args.disable_chunked_prefill: + invalid_prompt_length_map = {} + for request_idx, request in enumerate(requests): + if len(request.prompt_tokens) > context.max_tokens: + invalid_prompt_length_map[request_idx] = len(request.prompt_tokens) + assert not invalid_prompt_length_map, ( + "request idxs with prompts longer than context.max_tokens: " + ", ".join(f"{k}({v})" for k, v in invalid_prompt_length_map.items()) + ) # Inference engine. engine = DynamicInferenceEngine( @@ -423,8 +450,8 @@ def main(): ) # Print unique prompts + outputs. - if torch.distributed.get_rank() == 0: + if torch.distributed.get_rank() == 0: def escape_str(s): return s.replace("\n", "\\n") diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index 7b5de5c21f2..9e2b6bfa983 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -20,6 +20,10 @@ from megatron.training.arguments import parse_args from megatron.core import parallel_state +import logging + +logging.basicConfig(level=logging.INFO, force=True) + async def main( engine: DynamicInferenceEngine, requests: List[Request], diff --git a/examples/inference/gpt/utils.py b/examples/inference/gpt/utils.py index baa25787e83..0ea1f5a3df0 100644 --- a/examples/inference/gpt/utils.py +++ b/examples/inference/gpt/utils.py @@ -222,6 +222,9 @@ def arrival(r): if len(time_offsets) == 0: time_offsets = [0.0] + # Ensure first time is 0. + time_offsets = [to - time_offsets[0] for to in time_offsets] + # Truncate to num_requests. assert len(time_offsets) >= num_requests time_offsets = time_offsets[:num_requests] diff --git a/examples/post_training/modelopt/generation_server.py b/examples/post_training/modelopt/generation_server.py deleted file mode 100644 index b32cca0d73f..00000000000 --- a/examples/post_training/modelopt/generation_server.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - -"""Sample Generate""" -import os -import sys -import warnings -from functools import partial - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) -import os -import sys -from argparse import Namespace -from contextlib import nullcontext - -import torch - -from megatron.core import mpu -from megatron.core.inference.engines import AbstractEngine, StaticInferenceEngine -from megatron.core.inference.engines.abstract_engine import AbstractEngine -from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import ( - InferenceWrapperConfig, -) -from megatron.core.inference.sampling_params import SamplingParams -from megatron.core.inference.text_generation_controllers.text_generation_controller import ( - TextGenerationController, -) -from megatron.core.transformer.module import MegatronModule -from megatron.inference.text_generation import beam_search_and_post_process -from megatron.inference.text_generation.mcore_engine_server import ( - ModelInferenceWrapperServer, - run_mcore_engine, -) -from megatron.inference.text_generation_server import MegatronServer -from megatron.post_training.arguments import add_modelopt_args -from megatron.training import get_args, get_model, get_tokenizer, print_rank_0 -from megatron.training.checkpointing import load_checkpoint -from megatron.training.initialize import initialize_megatron - - -def get_inference_engine(args: Namespace, model: MegatronModule) -> AbstractEngine: - """Get the relevant backend for running inference - - This function will automatically choose the TRTLLMBackend when possible, and default to Mcore - backend if the user does not specify any backends. TRTLLMBackend is not implmented yet. - - Args: - args (Namespace): The user arguments parsed from command line - model (MegatronModule): The megatron model. - - Returns: - AbstractBackend: The chosen backend - """ - tokenizer = get_tokenizer() - - inference_wrapper_config = InferenceWrapperConfig( - hidden_size=args.hidden_size, - inference_batch_times_seqlen_threshold=args.inference_batch_times_seqlen_threshold, - fp32_residual_connection=args.fp32_residual_connection, - params_dtype=args.params_dtype, - padded_vocab_size=args.padded_vocab_size, - inference_max_seq_length=args.inference_max_seq_length, - inference_max_requests=args.inference_max_batch_size, - nccl_all_reduce_for_prefill=args.nccl_all_reduce_for_prefill, - ) - - inference_wrapped_model = ModelInferenceWrapperServer(model, inference_wrapper_config) - text_generation_controller = TextGenerationController( - inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer - ) - return StaticInferenceEngine( - text_generation_controller=text_generation_controller, - max_batch_size=args.inference_max_batch_size, - ) - - -def add_text_generate_args(parser): - """Adds text generation arguments to parser.""" - group = parser.add_argument_group(title='text generation') - group.add_argument( - "--port", type=int, default=5000, help='port for text generation server to run on' - ) - group.add_argument("--temperature", type=float, default=1.0, help='Sampling temperature.') - group.add_argument("--top_k", type=int, default=1, help='Top k sampling.') - group.add_argument("--top_p", type=float, default=0.0, help='Top p sampling.') - group.add_argument( - "--return-log-probs", - action='store_true', - default=True, - help='Return the log probabilities of the final output tokens', - ) - group.add_argument( - "--num-tokens-to-generate", - type=int, - default=30, - help='Number of tokens to generate for each prompt', - ) - group.add_argument( - "--prompts", - metavar='N', - type=str, - nargs='+', - help='Input prompts with each prompt within quotes and seperated by space', - ) - group.add_argument( - "--max-batch-size", - type=int, - default=None, - help='Deprecated in favor of `--inference-max-batch-size`', - ) - add_modelopt_args(parser) - return parser - - -@torch.inference_mode() -def main(model_provider: str = "gpt"): - """Runs the text generation server with the specified model provider.""" - initialize_megatron( - extra_args_provider=add_text_generate_args, - args_defaults={ - 'no_load_rng': True, - 'no_load_optim': True, - 'exit_on_missing_checkpoint': True, - }, - ) - args = get_args() - if args.num_layers_per_virtual_pipeline_stage is not None: - print("Interleaved pipeline schedule is not yet supported for text generation.") - exit() - print_rank_0("WARNING: Forcing exit_on_missing_checkpoint to True for text " "generation.") - args.exit_on_missing_checkpoint = True - - # Set up model and load checkpoint - load_context = nullcontext() - if args.fp8: - from transformer_engine.pytorch.fp8 import fp8_model_init - - load_context = fp8_model_init() - with load_context: - - from megatron.post_training.model_builder import modelopt_gpt_mamba_builder - from model_provider import model_provider as root_model_provider - if model_provider == "gpt": - model = get_model(partial(root_model_provider, modelopt_gpt_mamba_builder), wrap_with_ddp=False) - elif model_provider == "mamba": - pass - else: - raise ValueError(f"Invalid model provider {model_provider}") - - if args.load is not None: - _ = load_checkpoint(model, None, None, strict=False) - - assert len(model) == 1, "Above condition should have caught this" - model = model[0] - model.eval() - - if args.max_batch_size is not None: - assert args.inference_max_batch_size is not None - args.inference_max_batch_size = max(args.inference_max_batch_size, args.max_batch_size) - warnings.warn( - "`--max-batch-size` has been deprecated in favor of `--inference-max-requests`, " - f"setting maximum batch size to {args.inference_max_batch_size}" - ) - - inference_engine = get_inference_engine(args, model) - - if args.cuda_graph_impl == "local": - print(f"Running warmup for CUDA graphs...") - inference_engine.generate( - prompts=["Test prompt"], sampling_params=SamplingParams(num_tokens_to_generate=10) - ) - - if ( - mpu.is_pipeline_first_stage() - and mpu.get_tensor_model_parallel_rank() == 0 - and mpu.get_expert_model_parallel_rank() == 0 - ): - server = MegatronServer(inference_engine, args) - server.run("0.0.0.0", port=args.port) - - while True: - choice = torch.tensor(1, dtype=torch.long, device='cuda') - torch.distributed.broadcast(choice, 0) - if choice.item() == 0: - try: - run_mcore_engine(inference_engine) - except ValueError as ve: - pass - elif choice.item() == 1: - try: - beam_search_and_post_process( - inference_engine.text_generation_controller.inference_wrapped_model.model - ) - except ValueError as ve: - pass - - -if __name__ == "__main__": - main(model_provider="gpt") diff --git a/examples/post_training/modelopt/generation_server.sh b/examples/post_training/modelopt/generation_server.sh old mode 100644 new mode 100755 index a4d7ff2dada..9acd61f3d04 --- a/examples/post_training/modelopt/generation_server.sh +++ b/examples/post_training/modelopt/generation_server.sh @@ -14,11 +14,9 @@ if [ -z ${MLM_MODEL_CKPT} ]; then exit 1 fi -if [ -z ${DRAFT_LEN} ]; then - DRAFT_LEN=0 -fi +TOOLS_DIR="$(realpath ${SCRIPT_DIR}/../../../tools)" -${LAUNCH_SCRIPT} ${SCRIPT_DIR}/generation_server.py \ +${LAUNCH_SCRIPT} ${TOOLS_DIR}/run_text_generation_server.py \ ${MODEL_ARGS} \ --tensor-model-parallel-size ${TP} \ --expert-tensor-parallel-size ${ETP} \ diff --git a/megatron/core/datasets/megatron_tokenizer.py b/megatron/core/datasets/megatron_tokenizer.py index 224de24a9f9..08b602c4766 100644 --- a/megatron/core/datasets/megatron_tokenizer.py +++ b/megatron/core/datasets/megatron_tokenizer.py @@ -1,11 +1,14 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import json +import logging from abc import ABC, abstractmethod from collections import OrderedDict from typing import Any import numpy +logger = logging.getLogger(__name__) + class MegatronLegacyTokenizer(ABC): """Abstract class for tokenizer @@ -20,6 +23,12 @@ class MegatronLegacyTokenizer(ABC): """ def __init__(self, *tokenizer_paths: str, **tokenizer_options: Any): + # Deprecation warning + logger.warning( + "You’re using the legacy tokenizer system, which is deprecated " + "and will be removed in a future release. Please migrate to the new tokenizer system " + "(`megatron.core.tokenizers.MegatronTokenizer`)." + ) self.unique_identifiers = OrderedDict() self.unique_identifiers["class"] = type(self).__name__ self.unique_identifiers["tokenizer_path"] = list(tokenizer_paths) diff --git a/megatron/core/enums.py b/megatron/core/enums.py index c9a715519f9..fcca219badd 100644 --- a/megatron/core/enums.py +++ b/megatron/core/enums.py @@ -20,15 +20,17 @@ def encoder_and_decoder(self): class Fp8Recipe(str, enum.Enum): - """FP8 recipe names: delayed, tensorwise, mxfp8, blockwise.""" + """FP8 recipe names: delayed, tensorwise, mxfp8, blockwise, custom.""" delayed = "delayed" tensorwise = "tensorwise" mxfp8 = "mxfp8" blockwise = "blockwise" + custom = "custom" class Fp4Recipe(str, enum.Enum): - """FP4 recipe names: nvfp4.""" + """FP4 recipe names: nvfp4, custom.""" nvfp4 = "nvfp4" + custom = "custom" diff --git a/megatron/core/fp4_utils.py b/megatron/core/fp4_utils.py index eb02a4796b0..4f997f6be15 100644 --- a/megatron/core/fp4_utils.py +++ b/megatron/core/fp4_utils.py @@ -7,6 +7,7 @@ import torch from megatron.core.enums import Fp4Recipe +from megatron.core.fp8_utils import _get_custom_recipe from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.utils import is_te_min_version @@ -84,9 +85,11 @@ def get_fp4_recipe(config: TransformerConfig): Transformer Engine. Please make sure you are using TE version >= 2.7.0.dev0.""" ) + elif config.fp4_recipe == Fp4Recipe.custom: + fp4_recipe = _get_custom_recipe(config.fp4_quantizer_factory) else: raise ValueError( - "NVFP4BlockScaling is the only supported FP4 recipe. " + "NVFP4BlockScaling and custom are the only supported FP4 recipes. " "Please make sure you are using a compatible TE version >= 2.7.0.dev0." ) else: diff --git a/megatron/core/fp8_utils.py b/megatron/core/fp8_utils.py index 7c3591ae5f7..1c52e965cd7 100644 --- a/megatron/core/fp8_utils.py +++ b/megatron/core/fp8_utils.py @@ -2,14 +2,21 @@ """Utility functions related to FP8 that are used throughout Megatron core""" +import importlib import weakref from contextlib import nullcontext from functools import wraps -from typing import List, Optional +from typing import List, Optional, Union import torch -from megatron.core.enums import Fp8Recipe +from megatron.core.enums import Fp4Recipe, Fp8Recipe +from megatron.core.tensor_parallel import ( + ColumnParallelLinear, + RowParallelLinear, + gather_from_sequence_parallel_region, + reduce_scatter_to_sequence_parallel_region, +) from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.utils import get_te_version, is_te_min_version @@ -111,6 +118,53 @@ def dequantize_fp8_tensor(fp8_tensor: torch.Tensor) -> torch.Tensor: return fp8_tensor.from_float8() +def _resolve_callable_from_python_import_path(dotted_path: str): + """Resolve a Python import path like 'pkg.mod.func' to a callable. + + Raises ValueError with clear message on failure. + """ + if not isinstance(dotted_path, str) or not dotted_path: + raise ValueError( + "fp8_quantizer_factory must be a non-empty string with format 'pkg.mod.func'." + ) + + parts = dotted_path.rsplit(".", 1) + if len(parts) == 1: + raise ValueError(f"Invalid fp8_quantizer_factory '{dotted_path}'. Expected 'pkg.mod.func'.") + module_path, attr = parts[0], parts[1] + + try: + mod = importlib.import_module(module_path) + except Exception as exc: + raise ValueError( + f"Failed to import module '{module_path}' for fp8_quantizer_factory: {exc}" + ) from exc + + fn = getattr(mod, attr, None) + if fn is None: + raise ValueError( + f"Attribute '{attr}' not found in module '{module_path}' for fp8_quantizer_factory." + ) + if not callable(fn): + raise ValueError( + f"Resolved attribute '{module_path}.{attr}' is not callable for fp8_quantizer_factory." + ) + return fn + + +def _get_custom_recipe(quantizer_factory_python_path: str) -> Union[Fp8Recipe, Fp4Recipe]: + quantizer_factory = _resolve_callable_from_python_import_path(quantizer_factory_python_path) + try: + custom_recipe = transformer_engine.common.recipe.CustomRecipe(qfactory=quantizer_factory) + except AttributeError: + raise ValueError( + """CustomRecipe recipe is not available in this version of + Transformer Engine. Please make sure you are using TE version + >= 2.9.0.dev0.""" + ) + return custom_recipe + + def get_fp8_align_size(fp8_recipe: Fp8Recipe) -> int: """Get the alignment size required for fp8 GEMM.""" if fp8_recipe == Fp8Recipe.mxfp8: @@ -119,6 +173,27 @@ def get_fp8_align_size(fp8_recipe: Fp8Recipe) -> int: return 16 +def is_column_parallel_linear(module): + """Returns whether the given module is a ColumnParallelLinear layer.""" + if HAVE_TE and ( + isinstance(module, TEColumnParallelLinear) + or isinstance(module, TELayerNormColumnParallelLinear) + ): + return True + elif isinstance(module, ColumnParallelLinear): + return True + return False + + +def is_row_parallel_linear(module): + """Returns whether the given module is a RowParallelLinear layer.""" + if HAVE_TE and isinstance(module, TERowParallelLinear): + return True + elif isinstance(module, RowParallelLinear): + return True + return False + + """ The code below abstracts the functionalities needed for implementing "--fp8-param-gather" into several functions. It provides different implementations for each function based on different @@ -495,6 +570,8 @@ def get_fp8_recipe(config: TransformerConfig): fp8_recipe = transformer_engine.common.recipe.MXFP8BlockScaling( fp8_format=fp8_format ) + elif config.fp8_recipe == Fp8Recipe.custom: + fp8_recipe = _get_custom_recipe(config.fp8_quantizer_factory) else: raise ValueError( "Float8CurrentScaling, MXFP8BlockScaling, Float8BlockwiseScaling and " @@ -616,6 +693,18 @@ def padded_forward(input_tensor, *args, **kwargs): if not FP8GlobalStateManager.is_fp8_enabled(): return original_forward(input_tensor, *args, **kwargs) + # With sequence parallelism we need to all-gather before padding + # and reduce-scatter after unpadding + if is_sequence_parallel := getattr(module, "sequence_parallel", False): + if is_column_parallel_linear(module): + input_tensor = gather_from_sequence_parallel_region( + input_tensor, group=module.tp_group + ) + + # Disable sequence parallelism on the module because we are handling the + # all-gather and reduce-scatter externally + module.sequence_parallel = False + seq_len, batch_size, hidden_size = input_tensor.shape # Reshape to (S, B*H) to pad sequence dimension input_2d = input_tensor.reshape(seq_len, -1) @@ -641,6 +730,16 @@ def padded_forward(input_tensor, *args, **kwargs): unpadded_output_2d = _unpad_func(output_2d, [seq_len]) unpadded_output = unpadded_output_2d.reshape(seq_len, batch_size, output_hidden_size) + if is_sequence_parallel: + # Reduce-scatter after unpadding + if is_row_parallel_linear(module): + unpadded_output = reduce_scatter_to_sequence_parallel_region( + unpadded_output, group=module.tp_group + ) + + # Reset sequence parallelism flag on the module + module.sequence_parallel = True + if other_outputs: return (unpadded_output,) + other_outputs else: diff --git a/megatron/core/inference/async_stream.py b/megatron/core/inference/async_stream.py index 1bf8775e368..6c3242a13db 100644 --- a/megatron/core/inference/async_stream.py +++ b/megatron/core/inference/async_stream.py @@ -9,6 +9,7 @@ from typing import Any, AsyncGenerator, Callable, Optional, Type, Union from megatron.core.inference.inference_request import InferenceRequest +from megatron.core.utils import get_asyncio_loop STOP_ITERATION = Exception() @@ -20,12 +21,17 @@ class AsyncStream: Adopted from https://github.com/vllm-project/vllm/blob/eb881ed006ca458b052905e33f0d16dbb428063a/vllm/v1/engine/async_stream.py # pylint: disable=line-too-long """ - def __init__(self, request_id: int, cancel: Callable[[str], None]) -> None: + def __init__( + self, + request_id: int, + cancel: Callable[[str], None], + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: self._request_id = request_id self._cancel = cancel self._queue: asyncio.Queue = asyncio.Queue() self._finished = False - self._loop = asyncio.get_running_loop() + self._loop = get_asyncio_loop(loop) def put(self, item: Union[InferenceRequest, Exception]) -> None: """Adds a new value to the stream""" diff --git a/megatron/core/inference/contexts/attention_context/mamba_metadata.py b/megatron/core/inference/contexts/attention_context/mamba_metadata.py new file mode 100644 index 00000000000..e9cd99a6c48 --- /dev/null +++ b/megatron/core/inference/contexts/attention_context/mamba_metadata.py @@ -0,0 +1,106 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import torch + + +class MambaMetadata: + """Manages the metadata tensors required for Mamba layers during inference.""" + + def __init__(self, max_requests: int): + """ + Initializes the Mamba slot allocator. + + Args: + max_requests (int): The maximum number of concurrent requests. + """ + self.max_requests = max_requests + + # Metadata for mapping requests to slots in the static Mamba state buffer + self.request_to_mamba_state_idx = torch.full( + (self.max_requests,), -1, dtype=torch.int32, device=torch.cuda.current_device() + ) + + # Separate mapping used only for CUDA graph compatibility + self.request_to_mamba_state_idx_cudagraph_only = torch.full( + (self.max_requests,), -1, dtype=torch.int32, device=torch.cuda.current_device() + ) + + # Allocator for Mamba state slots + self.mamba_state_free_slots = torch.arange( + self.max_requests, dtype=torch.int32, device=torch.cuda.current_device() + ) + self.mamba_state_free_slot_count = self.max_requests + + def reset(self) -> None: + """ + Resets all Mamba states and frees all allocated slots. + """ + self.request_to_mamba_state_idx.fill_(-1) + self.request_to_mamba_state_idx_cudagraph_only.fill_(-1) + + # Re-initialize the free slot pool + self.mamba_state_free_slots = torch.arange( + self.max_requests, dtype=torch.int32, device=torch.cuda.current_device() + ) + self.mamba_state_free_slot_count = self.max_requests + + def reset_cudagraph_mapping(self) -> None: + """ + Resets only the CUDA graph mapping tensor. + """ + self.request_to_mamba_state_idx_cudagraph_only.fill_(-1) + + def update_cudagraph_mapping( + self, active_mamba_indices: torch.Tensor, num_active_requests: int + ) -> None: + """ + Updates the dedicated CUDA graph mapping tensor with the indices + of currently active requests. + + Args: + active_mamba_indices (Tensor): Tensor containing the Mamba slot indices + for active requests. + num_active_requests (int): The number of active requests. + """ + self.request_to_mamba_state_idx_cudagraph_only[0:num_active_requests] = active_mamba_indices + + def allocate_slot(self) -> int: + """ + Allocates a new slot for a request in the Mamba state buffers. + + Returns: + int: The index of the allocated slot. + Returns None if no slots are available. + """ + if self.mamba_state_free_slot_count == 0: + return None + + # Get a free slot + self.mamba_state_free_slot_count -= 1 + mamba_idx = self.mamba_state_free_slots[self.mamba_state_free_slot_count] + + return mamba_idx + + def free_slots(self, request_indices: torch.Tensor) -> None: + """ + Frees the Mamba state slots associated with the given request indices. + + Args: + request_indices (Tensor): A 1D tensor of request indices to free. + """ + # Get the Mamba state indices for finished requests + mamba_indices_to_free = self.request_to_mamba_state_idx[request_indices] + + # Filter out any invalid indices (e.g., -1) + mamba_indices_to_free = mamba_indices_to_free[mamba_indices_to_free != -1] + num_to_free = len(mamba_indices_to_free) + + if num_to_free > 0: + # Add the freed indices back to the free slot pool + start_idx = self.mamba_state_free_slot_count + end_idx = start_idx + num_to_free + self.mamba_state_free_slots[start_idx:end_idx] = mamba_indices_to_free + self.mamba_state_free_slot_count = end_idx + + # Invalidate the Mamba state index for the finished requests + self.request_to_mamba_state_idx[request_indices] = -1 diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index d6cc2598998..000b58200f8 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -23,9 +23,14 @@ from megatron.core.inference.utils import tensor_swap from megatron.core.models.common.embeddings.rope_utils import apply_rotary_pos_emb from megatron.core.package_info import __version__ as mcore_version +from megatron.core.ssm.mamba_hybrid_layer_allocation import ( + Symbols, + get_layer_maps_from_layer_type_list, +) from megatron.core.transformer import TransformerConfig from megatron.core.utils import divide as core_divide +from .attention_context.mamba_metadata import MambaMetadata from .attention_context.mha_metadata import GraphedMHAMetadata, NonGraphedMHAMetadata from .base_context import BaseInferenceContext from .dynamic_block_allocator import BlockAllocator @@ -227,8 +232,17 @@ class DynamicInferenceContext(BaseInferenceContext): where the cuda graph batch sizes range from 1 to `max_requests` (as computed below). Due to rounding, the actual number of cuda graphs may not equal this argument. - materialize_only_last_token_logits (bool): If True, only the last token logits - are materialized in the context. + materialize_only_last_token_logits (Optional[bool]): Whether to only + materialize logits for the last token. This should be set to False + if returning log probs. + layer_type_list (Optional[List[str]]): A list of strings that indicates + the layer type (Mamba / Attention / MLP) for each layer. + See `megatron/core/ssm/mamba_hybrid_layer_allocation.py` for the list + of symbols. This must be provided for hybrid models. + mamba_conv_states_shape: (Optional[Tuple[int]]): Mamba conv states shape per request. + This must be provided for hybrid models. + mamba_ssm_states_shape: (Optional[Tuple[int]]): Mamba ssm states shape per request. + This must be provided for hybrid models. use_cuda_graphs_for_non_decode_steps (bool): If True, use cuda graphs for non-decode engine steps. unified_memory_level (Optional[int]): Set unified memory usage within the @@ -259,7 +273,10 @@ def __init__( kv_lora_rank: Optional[int] = None, qk_pos_emb_head_dim: Optional[int] = None, num_cuda_graphs: Optional[int] = None, - materialize_only_last_token_logits: bool = True, + materialize_only_last_token_logits: Optional[bool] = True, + layer_type_list: Optional[List[str]] = None, + mamba_conv_states_shape: Optional[Tuple[int]] = None, + mamba_ssm_states_shape: Optional[Tuple[int]] = None, use_cuda_graphs_for_non_decode_steps: bool = True, use_flashinfer_fused_rope: bool = False, unified_memory_level: Optional[int] = 0, @@ -283,6 +300,41 @@ def __init__( tp_size = tensor_model_parallel_size hidden_size_per_attention_head = core_divide(projection_size, num_attention_heads) num_attention_heads_per_partition = core_divide(num_attention_heads, tp_size) + + # Mamba states. + self.is_hybrid_model = layer_type_list is not None and Symbols.MAMBA in layer_type_list + if self.is_hybrid_model: + assert ( + mamba_conv_states_shape is not None + ), "`mamba_conv_states_shape` must be specified for hybrid models" + assert ( + mamba_ssm_states_shape is not None + ), "`mamba_ssm_states_shape` must be specified for hybrid models" + assert ( + not use_cuda_graphs_for_non_decode_steps + ), "Non-decode CUDA graphs not yet supported for hybrid models" + + # For hybrid models, the layer map converts the global layer index to the + # corresponding attention layer index or Mamba layer index depending on the + # layer type. + attention_layer_map, mamba_layer_map, _ = get_layer_maps_from_layer_type_list( + layer_type_list + ) + self.num_attention_layers = len(attention_layer_map) + self.num_mamba_layers = len(mamba_layer_map) + self.layer_map = attention_layer_map | mamba_layer_map + else: + # The layer map is the identity function for pure Transformer models. + self.num_attention_layers = num_layers + self.num_mamba_layers = 0 + (mamba_conv_states_shape, mamba_ssm_states_shape) = (None, None) + self.layer_map = {i: i for i in range(self.num_attention_layers)} + + if self.num_attention_layers == 0: + raise NotImplementedError( + f"Using `DynamicInferenceContext` with no attention is not supported." + ) + # Block size tokens, bytes. dtype_size_bytes = params_dtype.itemsize self.block_size_tokens = block_size_tokens @@ -297,24 +349,38 @@ def __init__( self.block_size_bytes = ( dtype_size_bytes * 2 # key, value - * num_layers + * self.num_attention_layers * self.block_size_tokens * num_attention_heads_per_partition * hidden_size_per_attention_head ) + assert self.block_size_bytes > 0 # Adjust buffer to be a multiple of block size. buffer_size_bytes = int(buffer_size_gb * 1024**3) buffer_size_bytes_rem = buffer_size_bytes % self.block_size_bytes buffer_size_bytes = buffer_size_bytes - buffer_size_bytes_rem - # Compute max_requets, max_tokens from buffer size and overflow factor. + mamba_states_memory_per_request = 0 + if self.is_hybrid_model: + mamba_states_memory_per_request += math.prod(mamba_conv_states_shape) + mamba_states_memory_per_request += math.prod(mamba_ssm_states_shape) + mamba_states_memory_per_request *= self.num_mamba_layers + mamba_states_memory_per_request *= dtype_size_bytes + + # Compute max_requets, max_tokens from buffer size, overflow factor, and Mamba state size. def bytes_to_max_requests_and_tokens(n_bytes): - n_tokens = n_bytes / self.block_size_bytes * self.block_size_tokens - n_requests = n_tokens / max_sequence_length - return self.round_up_requests(int(n_requests), tp_size=tp_size), self.round_up_tokens( - int(n_tokens), tp_size=tp_size + bytes_per_token = self.block_size_bytes / self.block_size_tokens + cost_per_request_bytes = ( + mamba_states_memory_per_request + max_sequence_length * bytes_per_token ) + # TODO(ksanthanam): Leave room for an extra request in the event of padding + # for non-decode CUDA graphs + n_requests = n_bytes / cost_per_request_bytes + n_tokens = n_requests * max_sequence_length + n_requests = self.round_up_requests(int(n_requests), tp_size=tp_size) + n_tokens = self.round_up_tokens(int(n_tokens), tp_size=tp_size) + return n_requests, n_tokens self.max_requests, self.max_tokens = bytes_to_max_requests_and_tokens(buffer_size_bytes) if buffer_overflow_factor is not None: @@ -339,7 +405,6 @@ def bytes_to_max_requests_and_tokens(n_bytes): # Initialize context state. self.params_dtype = params_dtype - self.num_layers = num_layers self.max_sequence_length = max_sequence_length # Unified memory. @@ -390,8 +455,11 @@ def bytes_to_max_requests_and_tokens(n_bytes): self.token_to_position_in_request = torch.empty_like(self.token_to_input_ids) self.token_to_local_position_within_kv_block = torch.empty_like(self.token_to_input_ids) - # Calculate the total number of blocks available in the buffer - block_count_total = buffer_size_bytes // self.block_size_bytes + # Calculate the total number of chunks available in the buffer + total_mamba_states_memory = mamba_states_memory_per_request * self.max_requests + block_count_total = ( + max(0, buffer_size_bytes - total_mamba_states_memory) // self.block_size_bytes + ) # Memory buffer. ctx_manager = ( @@ -402,7 +470,12 @@ def bytes_to_max_requests_and_tokens(n_bytes): with ctx_manager: if cache_mla_latent: self.memory_buffer = torch.full( - (self.num_layers, block_count_total, self.block_size_tokens, kv_reduced_dim), + ( + self.num_attention_layers, + block_count_total, + self.block_size_tokens, + kv_reduced_dim, + ), -1, dtype=self.params_dtype, device=torch.cuda.current_device(), @@ -411,7 +484,7 @@ def bytes_to_max_requests_and_tokens(n_bytes): self.memory_buffer = torch.full( ( 2, # key and value - self.num_layers, + self.num_attention_layers, block_count_total, self.block_size_tokens, num_attention_heads_per_partition, @@ -516,14 +589,34 @@ def bytes_to_max_requests_and_tokens(n_bytes): block_count_total=block_count_total, gtd_block_count=self.gtd_block_count ) + # Optional state tensors for hybrid models + if self.is_hybrid_model: + self.mamba_metadata = MambaMetadata(max_requests=self.max_requests) + + with ctx_manager: + self.mamba_conv_states = torch.zeros( + (self.num_mamba_layers, self.max_requests) + mamba_conv_states_shape, + dtype=self.params_dtype, + device=torch.cuda.current_device(), + ) + self.mamba_ssm_states = torch.zeros( + (self.num_mamba_layers, self.max_requests) + mamba_ssm_states_shape, + dtype=self.params_dtype, + device=torch.cuda.current_device(), + ) + + else: + self.mamba_metadata = None + # Store the dummy block idx reference for convenience self.dummy_block_idx = self.block_allocator.dummy_block_idx # Deal with chunked prefill self.chunked_prefill_request_id = -1 - # Reset attention state. + # Reset attention and Mamba state. self.reset_attention_state() + self.reset_mamba_state() if use_flashinfer_fused_rope is True: assert HAVE_FLASHINFER, "flashinfer is not installed" @@ -628,7 +721,8 @@ def is_decode_only(self) -> bool: """Test if all active requests are in decode phase. For a request in prefill phase active_tokens = query length - Once the request moves to decode phase active tokens is 1 for that request. So if all active requests are in decode phase, they will be equal to active token count. + Once the request moves to decode phase active tokens is 1 for that request. + So if all active requests are in decode phase, they will be equal to active token count. """ total_active_requests = self.total_request_count - self.paused_request_count return total_active_requests == self.active_token_count @@ -664,11 +758,7 @@ def get_max_sequence_lengths(self) -> Tensor: def get_active_request_count(self): """Returns the current number of active requests.""" - active_sequence_lengths = self.get_active_sequence_lengths() - max_sequence_lengths = self.get_max_sequence_lengths() - active_requests_mask = torch.less(active_sequence_lengths, max_sequence_lengths).byte() - active_request_count = (active_requests_mask == 1).sum().item() - return active_request_count + return self.total_request_count - self.paused_request_count def append_key_value_cache(self, layer_number: int, key: Tensor, value: Tensor) -> None: """Append to KV cache. @@ -678,10 +768,12 @@ def append_key_value_cache(self, layer_number: int, key: Tensor, value: Tensor) key (Tensor): Key tensor. value (Tensor): Value tensor. """ + attention_layer_number = self.layer_map[layer_number - 1] + if triton_append_key_value_cache is not None and not self.cache_mla_latent: # currently does not support MLA latent cache return triton_append_key_value_cache( - layer_number=layer_number, + layer_number=attention_layer_number, key=key, value=value, memory_buffer=self.memory_buffer, @@ -706,14 +798,14 @@ def append_key_value_cache(self, layer_number: int, key: Tensor, value: Tensor) if self.cache_mla_latent: # We pass the kv_concat as the key in cache_mla_latent kv_concat = key - self.memory_buffer[layer_number - 1, block_idx, local_kv_seq_idx] = kv_concat[ + self.memory_buffer[attention_layer_number, block_idx, local_kv_seq_idx] = kv_concat[ : self.padded_active_token_count ] else: - self.memory_buffer[0, layer_number - 1, block_idx, local_kv_seq_idx] = key[ + self.memory_buffer[0, attention_layer_number, block_idx, local_kv_seq_idx] = key[ : self.padded_active_token_count ] - self.memory_buffer[1, layer_number - 1, block_idx, local_kv_seq_idx] = value[ + self.memory_buffer[1, attention_layer_number, block_idx, local_kv_seq_idx] = value[ : self.padded_active_token_count ] @@ -727,19 +819,30 @@ def key_value_cache(self, layer_number: int) -> Tuple[Tensor, Tensor]: (Tuple[Tensor, Tensor]) The key and value pointer tensors that point to blocks within the block-level memory buffer. """ + attention_layer_number = self.layer_map[layer_number - 1] if self.cache_mla_latent: return ( - self.memory_buffer[layer_number - 1], + self.memory_buffer[attention_layer_number], None, self.active_attn_metadata["mha_metadata"].state_data["block_table"], ) else: return ( - self.memory_buffer[0, layer_number - 1], - self.memory_buffer[1, layer_number - 1], + self.memory_buffer[0, attention_layer_number], + self.memory_buffer[1, attention_layer_number], self.active_attn_metadata["mha_metadata"].state_data["block_table"], ) + def mamba_states_cache(self, layer_number: int) -> Tuple[Tensor, Tensor]: + """Returns the Mamba state tensors for the given layer.""" + assert self.is_hybrid_model, "Only hybrid models have Mamba state tensors" + + mamba_layer_number = self.layer_map[layer_number - 1] + conv_state = self.mamba_conv_states[mamba_layer_number] + ssm_state = self.mamba_ssm_states[mamba_layer_number] + + return (conv_state, ssm_state) + def apply_fused_qk_rotary_emb( self, query: Tensor, key: Tensor, cos_sin_emb: Tensor, config: TransformerConfig ) -> Tuple[Tensor, Tensor]: @@ -854,6 +957,16 @@ def reset_attention_state(self) -> None: attn_metadata.reset() self.active_attn_metadata = None + if self.is_hybrid_model: + self.mamba_metadata.reset_cudagraph_mapping() + + def reset_mamba_state(self) -> None: + """Reset state used within Mamba layers.""" + if self.is_hybrid_model: + self.mamba_conv_states.fill_(0) + self.mamba_ssm_states.fill_(0) + self.mamba_metadata.reset() + def using_cuda_graph_this_step(self) -> bool: """Returns True if cuda graphs are being used for this step.""" has_cuda_graphs = self.cuda_graph_token_counts is not None @@ -977,6 +1090,17 @@ def initialize_attention_state( ) # All attention metadata calculations are now handled by MHAMetadata.update() + # Create Mamba state block table if it's a hybrid model + if self.is_hybrid_model: + active_mamba_indices = self.mamba_metadata.request_to_mamba_state_idx[ + self.paused_request_count : self.total_request_count + ] + + if self.is_decode_only() or self.using_cuda_graph_this_step(): + self.mamba_metadata.update_cudagraph_mapping( + active_mamba_indices, self.total_request_count - self.paused_request_count + ) + def reset(self) -> None: """Reset entire context. @@ -1018,15 +1142,13 @@ def reset(self) -> None: # Reset available block count. self.reset_attention_state() + self.reset_mamba_state() self.block_allocator.reset() self.request_to_kv_block_ids.fill_(-1) # Reset chunked prefill state self.chunked_prefill_request_id = -1 - # Reset chunked prefill state - self.chunked_prefill_request_id = -1 - def current_input_and_position_ids( self, *, num_warmup_tokens: Optional[int] = None ) -> Tuple[Tensor, Tensor]: @@ -1198,6 +1320,18 @@ def add_request(self, req: DynamicInferenceRequest, chunk_length: Optional[int] self.token_to_local_position_within_kv_block[ self.active_token_count : self.active_token_count + chunk_length ] = (token_offset_range % self.block_size_tokens) + + if self.is_hybrid_model and not is_chunked_prefill: + # Allocate a slot for Mamba states + mamba_idx = self.mamba_metadata.allocate_slot() + if mamba_idx is None: + raise ContextOverflowError(req.request_id, "No Mamba slots available") + + # Initialize the allocated Mamba state + self.mamba_conv_states[:, mamba_idx] = 0.0 + self.mamba_ssm_states[:, mamba_idx] = 0.0 + self.mamba_metadata.request_to_mamba_state_idx[self.total_request_count] = mamba_idx + self.active_token_count += chunk_length self.total_request_count += 0 if req.finished_chunk_token_count > 0 else 1 @@ -1216,6 +1350,11 @@ def _move_book_keeping_tensors(self, src_idxs, dst_idxs, next_tokens): self.request_last_kv_block_id[dst_idxs] = self.request_last_kv_block_id[src_idxs] self.request_last_kv_block_offset[dst_idxs] = self.request_last_kv_block_offset[src_idxs] + if self.is_hybrid_model: + self.mamba_metadata.request_to_mamba_state_idx[dst_idxs] = ( + self.mamba_metadata.request_to_mamba_state_idx[src_idxs] + ) + def _swap_book_keeping_tensors(self, src_idxs, dst_idxs, next_tokens): """ Swaps all the relevent booking tensors with src idxs to dst idxs @@ -1230,6 +1369,9 @@ def _swap_book_keeping_tensors(self, src_idxs, dst_idxs, next_tokens): tensor_swap(self.request_last_kv_block_id, src_idxs, dst_idxs) tensor_swap(self.request_last_kv_block_offset, src_idxs, dst_idxs) + if self.is_hybrid_model: + tensor_swap(self.mamba_metadata.request_to_mamba_state_idx, src_idxs, dst_idxs) + # TODO: see if we can compile this function def update_requests(self, active_requests_mask: Tensor, new_tokens: Tensor) -> Tensor: """Update context state after calling engine.step(). @@ -1301,10 +1443,17 @@ def update_requests(self, active_requests_mask: Tensor, new_tokens: Tensor) -> T non_zero_values_in_kv_memory = kv_blocks_assigned[kv_blocks_assigned != -1] self.block_allocator.release_memory_blocks(non_zero_values_in_kv_memory) + if self.is_hybrid_model: + self.mamba_metadata.free_slots(finished_idxs) + # Reset request/token counts. self.request_to_kv_block_ids.fill_(-1) self.total_request_count = 0 self.active_token_count = 0 + + # Reset Mamba state. + self.reset_mamba_state() + return # 3. Concatenate the paused tokens to the active tokens if present. @@ -1332,6 +1481,10 @@ def update_requests(self, active_requests_mask: Tensor, new_tokens: Tensor) -> T # and updates it instead of the original tensor. self.request_to_kv_block_ids[finished_idxs] = -1 + if self.is_hybrid_model: + # Get the Mamba state indices for finished requests and free them + self.mamba_metadata.free_slots(finished_idxs) + if active_request_count > 0: finished_idxs_on_left = ( torch.nonzero(active_requests_mask[:active_request_count] == 0, as_tuple=True)[ @@ -1351,8 +1504,10 @@ def update_requests(self, active_requests_mask: Tensor, new_tokens: Tensor) -> T next_tokens=next_tokens, ) - # Reset block ids for recently moved requests. + # Reset chunk ids for recently moved requests. self.request_to_kv_block_ids[active_idxs_on_right] = -1 + if self.is_hybrid_model: + self.mamba_metadata.request_to_mamba_state_idx[active_idxs_on_right] = -1 # 5. We identify requests that require a new block and add them to the paused requests (i.e move them left) :- # a) Put requests that have filled their current block and require a new one in a pause state temporarily @@ -1450,6 +1605,7 @@ def update_requests(self, active_requests_mask: Tensor, new_tokens: Tensor) -> T # 7. We make changes to the request book keeping tesnsors and setup the tokens for next iteration self.total_request_count = active_request_count + self.paused_request_count + # All these active requests are in decode phase, so they need only 1 token per request self.active_token_count = active_request_count # Always the first section of token input ids are only used. diff --git a/megatron/core/inference/contexts/fused_kv_append_kernel.py b/megatron/core/inference/contexts/fused_kv_append_kernel.py index 2078878c8f4..db1eed456e1 100644 --- a/megatron/core/inference/contexts/fused_kv_append_kernel.py +++ b/megatron/core/inference/contexts/fused_kv_append_kernel.py @@ -119,8 +119,8 @@ def triton_append_key_value_cache( _, num_heads, h_dim = key.shape - key_cache = memory_buffer[0, layer_number - 1] - value_cache = memory_buffer[1, layer_number - 1] + key_cache = memory_buffer[0, layer_number] + value_cache = memory_buffer[1, layer_number] key_to_cache = key[:n_tokens] value_to_cache = value[:n_tokens] diff --git a/megatron/core/inference/data_parallel_inference_coordinator.py b/megatron/core/inference/data_parallel_inference_coordinator.py index ea0560183d8..0045d5947a1 100644 --- a/megatron/core/inference/data_parallel_inference_coordinator.py +++ b/megatron/core/inference/data_parallel_inference_coordinator.py @@ -1,6 +1,8 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import faulthandler import logging +import signal from collections import deque from itertools import cycle from multiprocessing import Event @@ -23,6 +25,11 @@ except: HAVE_MSGPACK = False +# Register faulthandler to emit stack traces upon process kill. +faulthandler.enable() +faulthandler.register(signal.SIGTERM, all_threads=False, chain=True) +faulthandler.register(signal.SIGINT, all_threads=False, chain=True) + class DataParallelInferenceCoordinator: """ diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index bcde4f9894d..4bff4f85fa8 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -33,8 +33,8 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) -from megatron.core.inference.utils import Counter -from megatron.core.utils import get_asyncio_loop +from megatron.core.inference.utils import Counter, await_process_event +from megatron.core.utils import get_asyncio_loop, trace_async_exceptions try: from tqdm import tqdm @@ -293,7 +293,11 @@ def create_cuda_graphs(self, reset_context: bool = True): self.capture_stats = capture_stats async def start_listening_to_data_parallel_coordinator( - self, inference_coordinator_port: int, launch_inference_coordinator: bool = True + self, + inference_coordinator_port: int, + launch_inference_coordinator: bool = True, + *, + loop: Optional[asyncio.AbstractEventLoop] = None, ): """Initializes ZMQ communication to connect the engine with an inference coordinator. @@ -407,12 +411,14 @@ async def start_listening_to_data_parallel_coordinator( torch.distributed.barrier(parallel_state.get_tensor_model_parallel_group()) if launch_inference_coordinator and torch.distributed.get_rank() == 0: - coordinator_ready_event.wait() + await await_process_event(coordinator_ready_event, self.inference_coordinator_process) logging.info("Inference co-ordinator is ready to receive requests!") # Finally run the engine infinite loop - self.engine_loop_task = asyncio.create_task(self.run_engine_with_coordinator()) + loop = get_asyncio_loop(loop) + self.engine_loop_task = loop.create_task(self.run_engine_with_coordinator(loop=loop)) + @trace_async_exceptions async def _notify_cond_for_new_request(self): """Helper function to notify condition variable when a new request is added.""" async with self._cond: @@ -466,7 +472,7 @@ def _add_request( self.waiting_request_ids.append(request_id) # Create a new asyncio Future to notify the user when the request has completed. - self.request_completion_futures[request_id] = asyncio.Future() + self.request_completion_futures[request_id] = self._loop.create_future() return self.request_completion_futures[request_id] def add_request( @@ -641,7 +647,7 @@ def schedule_non_chunked_prefill(self): if request_can_be_added and request_tokens_can_be_added and kv_cache_available: self.context.add_request(req) self._loop.call_soon_threadsafe( - asyncio.create_task, self._notify_cond_for_new_request() + self._loop.create_task, self._notify_cond_for_new_request() ) req.remaining_prompt_tokens = req.remaining_prompt_tokens.new_empty(0) req.add_event_add() @@ -702,7 +708,7 @@ def schedule_chunked_prefill(self): # is_continuing_chunked_prefill is True if we are scheduling next # chunk of a existing chunked prefill request - is_continuing_chunked_prefill = self.context.chunked_prefill_request_id > 0 + is_continuing_chunked_prefill = self.context.chunked_prefill_request_id >= 0 # Use remaining prompt tokens for scheduling decisions remaining_len = len(req.remaining_prompt_tokens) @@ -720,7 +726,7 @@ def schedule_chunked_prefill(self): self.context.chunked_prefill_request_id = -1 self.context.add_request(req) self._loop.call_soon_threadsafe( - asyncio.create_task, self._notify_cond_for_new_request() + self._loop.create_task, self._notify_cond_for_new_request() ) req.remaining_prompt_tokens = req.remaining_prompt_tokens.new_empty(0) req.add_event_add() @@ -732,7 +738,7 @@ def schedule_chunked_prefill(self): chunk_length = self.context.max_tokens - self.context.active_token_count self.context.add_request(req, chunk_length=chunk_length) self._loop.call_soon_threadsafe( - asyncio.create_task, self._notify_cond_for_new_request() + self._loop.create_task, self._notify_cond_for_new_request() ) self.context.chunked_prefill_request_id = req.request_id req.remaining_prompt_tokens = req.remaining_prompt_tokens[chunk_length:] @@ -939,7 +945,7 @@ def generate( result = self.step_modern() finished_requests_list.extend(result["finished_requests"]) - # Ensure requests are returned in the same order they were passed in. + # Ensure requests are returned in the same order they were passed in finished_requests_list.sort(key=lambda x: x.request_id) return finished_requests_list @@ -1039,8 +1045,12 @@ def stop(self): self.zmq_context.term() parallel_state.destroy_model_parallel() - async def run_engine(self, *, verbose: Optional[bool] = False): + @trace_async_exceptions + async def run_engine( + self, *, loop: Optional[asyncio.AbstractEventLoop] = None, verbose: Optional[bool] = False + ): """Continually steps the engine asynchronously.""" + self._loop = get_asyncio_loop(loop) try: while True: # Wait until there are active requests before proceeding. @@ -1054,8 +1064,12 @@ async def run_engine(self, *, verbose: Optional[bool] = False): except asyncio.CancelledError: pass - async def run_engine_with_coordinator(self, *, verbose: Optional[bool] = False): + @trace_async_exceptions + async def run_engine_with_coordinator( + self, *, loop: Optional[asyncio.AbstractEventLoop] = None, verbose: Optional[bool] = False + ): """Continually steps the engine asynchronously.""" + self._loop = get_asyncio_loop(loop) try: while True: self.schedule_requests() diff --git a/megatron/core/inference/engines/static_engine.py b/megatron/core/inference/engines/static_engine.py index d084528b8f2..dc86eb775f9 100644 --- a/megatron/core/inference/engines/static_engine.py +++ b/megatron/core/inference/engines/static_engine.py @@ -17,6 +17,7 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.utils import get_asyncio_loop try: from tqdm import tqdm @@ -217,11 +218,6 @@ def generate_using_dynamic_engine( generated tokens, texts and log probs if required """ assert hasattr(self, 'dynamic_engine'), "Dynamic engine not initialized" - try: - loop = asyncio.get_running_loop() - except RuntimeError: # 'RuntimeError: There is no current event loop...' - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) if common_inference_params: sampling_params = common_inference_params @@ -385,8 +381,8 @@ def _wrapped_run_engine(self, cuda_device): torch.cuda.set_device(cuda_device) self.run_engine() - async def run_engine_async(self): + async def run_engine_async(self, loop: Optional[asyncio.AbstractEventLoop] = None): """Runs the engine asynchronously using asyncio""" - loop = asyncio.get_running_loop() + loop = get_asyncio_loop(loop) await loop.run_in_executor(None, self._wrapped_run_engine, torch.cuda.current_device()) diff --git a/megatron/core/inference/inference_client.py b/megatron/core/inference/inference_client.py index 59b9144a207..53daac091b0 100644 --- a/megatron/core/inference/inference_client.py +++ b/megatron/core/inference/inference_client.py @@ -8,6 +8,7 @@ from megatron.core.inference.inference_request import DynamicInferenceRequest from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.utils import get_asyncio_loop, trace_async_exceptions from .headers import Headers @@ -103,10 +104,11 @@ def add_request( payload_serialized = msgpack.packb(payload, use_bin_type=True) self.socket.send(payload_serialized) assert request_id not in self.completion_futures - self.completion_futures[request_id] = asyncio.get_event_loop().create_future() + self.completion_futures[request_id] = get_asyncio_loop().create_future() self.request_submission_times[request_id] = time.perf_counter() return self.completion_futures[request_id] + @trace_async_exceptions async def _listen_for_completed_requests(self): """ Listens for completed inference requests from the coordinator. diff --git a/megatron/core/inference/sampling_params.py b/megatron/core/inference/sampling_params.py index a64e2e56775..e215b3f134b 100644 --- a/megatron/core/inference/sampling_params.py +++ b/megatron/core/inference/sampling_params.py @@ -21,6 +21,7 @@ class SamplingParams: top_k: int = 0 top_p: float = 0.0 return_log_probs: bool = False + skip_prompt_log_probs: bool = False return_segments: bool = False # Whether to return individually detokenized tokens num_tokens_to_generate: int = 30 num_tokens_total: Optional[int] = None # Cannot set both this and num_tokens_to_generate diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index ef092173c6a..2bda1425710 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -626,11 +626,8 @@ def _dynamic_step_calculate_log_probs( return_log_probs = False for sampling_params, mask in active_sampling_map: if sampling_params.return_log_probs: - skip_prompt_log_probs_for_dynamic_inference = getattr( - sampling_params, "skip_prompt_log_probs_for_dynamic_inference", False - ) assert ( - skip_prompt_log_probs_for_dynamic_inference + sampling_params.skip_prompt_log_probs or materialize_only_last_token_logits is False ), "Materialize only last token logits must be false for returning log probs" return_log_probs = True @@ -762,10 +759,12 @@ async def async_generate_output_tokens_dynamic_batch( @torch.inference_mode() def generate_output_tokens_dynamic_batch( - self, active_sampling_map: List[Tuple[SamplingParams, List[int]]] + self, + active_sampling_map: List[Tuple[SamplingParams, List[int]]], + loop: Optional[asyncio.AbstractEventLoop] = None, ) -> Optional[Dict]: """Synchronous wrapper for `self.async_generate_output_tokens_dynamic_batch.""" - loop = get_asyncio_loop() + loop = get_asyncio_loop(loop) return loop.run_until_complete( self.async_generate_output_tokens_dynamic_batch(active_sampling_map) ) diff --git a/megatron/core/inference/utils.py b/megatron/core/inference/utils.py index 985042f31e3..d58f3c3a652 100644 --- a/megatron/core/inference/utils.py +++ b/megatron/core/inference/utils.py @@ -1,5 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +import asyncio +import multiprocessing + import torch from megatron.core.transformer.moe.moe_layer import MoELayer @@ -133,3 +136,28 @@ def tensor_swap(x, src_idxs, dst_idxs): Swap x[src_idxs] and x[dst_idxs] """ x[dst_idxs], x[src_idxs] = x[src_idxs], x[dst_idxs] + + +async def await_process_event( + event: multiprocessing.Event, process: multiprocessing.Process, timeout: float = 1.0 +) -> None: + """Repeatedly wait for a multiprocessing event to be set, aborting upon process failure. + + Note that the timeout in this function is only for checking process liveness. + Its value should be set to a relatively high number. The only problem a high timeout + introduces is that an error is raised slighly later. + The timeout does not have any effect on the event-waiting, only on process failure detection. + + Args: + event: The multiprocessing event to wait on. + process: The process to monitor for failure. + timeout: The timeout for each wait iteration in seconds. + """ + while True: + signal = await asyncio.to_thread(event.wait, timeout) + if signal: + return + if not process.is_alive(): + raise RuntimeError( + f"Process {process.name} (pid {process.pid}) has exited unexpectedly." + ) diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index a1156012106..e840fca99b3 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -611,8 +611,6 @@ def _postprocess( # Perform the sequence parallel gather here instead of after the output layer # because we need to slice the last token logits from the full view of the # packed logits across all requests. - # TODO(ksanthanam): Make the equivalent change in the `MambaModel` code after - # merging in !3722. hidden_states = gather_from_sequence_parallel_region( hidden_states, group=self.pg_collection.tp ) @@ -620,7 +618,7 @@ def _postprocess( sequence_parallel_override = True # Reshape [B, 1, H] to [1, B, H] → extract each sample’s true last‐token hidden - # state ([B, H]) → unsqueeze back to [1, B, H] + # state ([B, H]) → unsqueeze back to [B, 1, H] # (so that the output layer, which expects S×B×H, receives only the final token) hidden_states = inference_context.last_token_logits( hidden_states.squeeze(1).unsqueeze(0) diff --git a/megatron/core/models/mamba/mamba_model.py b/megatron/core/models/mamba/mamba_model.py index fb3df5e23f2..378cf7e47d6 100644 --- a/megatron/core/models/mamba/mamba_model.py +++ b/megatron/core/models/mamba/mamba_model.py @@ -12,6 +12,7 @@ from megatron.core.models.common.language_module.language_module import LanguageModule from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.quantization.utils import get_quant_config_or_none +from megatron.core.tensor_parallel import gather_from_sequence_parallel_region from megatron.core.transformer import TransformerConfig from megatron.core.transformer.enums import ModelType from megatron.core.transformer.spec_utils import ModuleSpec, build_module @@ -244,13 +245,41 @@ def forward( if self.share_embeddings_and_output_weights: output_weight = self.shared_embedding_or_output_weight() + sequence_parallel_override = False if in_inference_mode and inference_context.materialize_only_last_token_logits: - hidden_states = hidden_states[-1, :, :].unsqueeze(0) + if inference_context.is_static_batching(): + hidden_states = hidden_states[-1:, :, :] + else: + if self.output_layer.sequence_parallel: + # Perform the sequence parallel gather here instead of after the output layer + # because we need to slice the last token logits from the full view of the + # packed logits across all requests. + hidden_states = gather_from_sequence_parallel_region( + hidden_states, group=self.pg_collection.tp + ) + self.output_layer.sequence_parallel = False + sequence_parallel_override = True + + # Reshape [B, 1, H] to [1, B, H] → extract each sample’s true last‐token hidden + # state ([B, H]) → unsqueeze back to [B, 1, H] + # (so that the output layer, which expects S×B×H, receives only the final token) + hidden_states = inference_context.last_token_logits( + hidden_states.squeeze(1).unsqueeze(0) + ).unsqueeze(1) logits, _ = self.output_layer( hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output ) + # Restore sequence parallel execution to the output layer if necessary. + if sequence_parallel_override: + assert ( + in_inference_mode + and inference_context.is_dynamic_batching() + and inference_context.materialize_only_last_token_logits + ) + self.output_layer.sequence_parallel = True + if labels is None: # [s b h] => [b s h] return logits.transpose(0, 1).contiguous() diff --git a/megatron/core/safe_globals.py b/megatron/core/safe_globals.py index 94a05e03c41..d2baed2a4a0 100755 --- a/megatron/core/safe_globals.py +++ b/megatron/core/safe_globals.py @@ -1,6 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. from argparse import Namespace +from io import BytesIO from pathlib import PosixPath from types import SimpleNamespace @@ -26,6 +27,7 @@ RerunDiagnostic, RerunMode, RerunState, + BytesIO, ] diff --git a/megatron/core/ssm/mamba_block.py b/megatron/core/ssm/mamba_block.py index 5f426ea347b..1bcadd0af10 100644 --- a/megatron/core/ssm/mamba_block.py +++ b/megatron/core/ssm/mamba_block.py @@ -9,7 +9,7 @@ from contextlib import nullcontext from dataclasses import dataclass from functools import partial -from typing import Optional, Union +from typing import Optional, Tuple, Union import torch from torch import Tensor, nn @@ -148,7 +148,7 @@ def __init__( self.hybrid_mlp_ratio = hybrid_mlp_ratio self.hybrid_override_pattern = hybrid_override_pattern - layer_type_list = allocate_layers( + self.layer_type_list = allocate_layers( self.config.num_layers, self.hybrid_attention_ratio, self.hybrid_mlp_ratio, @@ -157,12 +157,12 @@ def __init__( pp_layer_offset = 0 if self.pp_group.size() > 1: - pp_layer_offset, layer_type_list = self._select_layers_for_pipeline_parallel( - layer_type_list + pp_layer_offset, self.layer_type_list = self._select_layers_for_pipeline_parallel( + self.layer_type_list ) self.layers = nn.ModuleList() - for i, layer_type in enumerate(layer_type_list): + for i, layer_type in enumerate(self.layer_type_list): fp8_init_context = get_fp8_context(self.config, i + pp_layer_offset, is_init=True) with fp8_init_context: if layer_type == LayerSymbols.MAMBA: @@ -226,22 +226,6 @@ def _select_layers_for_pipeline_parallel(self, layer_type_list): return offset, selected_list - def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None): - """ - Allocate inference cache for each layer. - - Args: - batch_size (int): The batch size to use for inference. - max_seqlen (int): The maximum sequence length to use - for inference. - dtype (optional): The data type to use for allocation. - Defaults to the data type of the model. - """ - return { - i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype) - for i, layer in enumerate(self.layers) - } - def set_input_tensor(self, input_tensor: Tensor): """Set input tensor to be used instead of forward()'s input. @@ -252,6 +236,16 @@ def set_input_tensor(self, input_tensor: Tensor): forward_step_func""" self.input_tensor = input_tensor + def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int]]]: + """ + Returns the Mamba conv and ssm states shapes per input sequence + if this block contains Mamba layers (this may not be the case with PP > 1). + """ + for layer_type, layer in zip(self.layer_type_list, self.layers): + if layer_type == LayerSymbols.MAMBA: + return layer.mamba_state_shapes_per_request() + return None + def forward( self, hidden_states: Union[Tensor, WrappedTensor], @@ -289,10 +283,7 @@ def forward( if isinstance(hidden_states, WrappedTensor): hidden_states = hidden_states.unwrap() - if inference_context: - assert ( - inference_context.is_static_batching() - ), "Mamba currently does not support dynamic inference batching." + if inference_context and inference_context.is_static_batching(): # NOTE(bnorick): match BaseInferenceContext attributes for # mamba_ssm.utils.generation.BaseInferenceContext, # this hack supports eval diff --git a/megatron/core/ssm/mamba_hybrid_layer_allocation.py b/megatron/core/ssm/mamba_hybrid_layer_allocation.py index 26972b5454b..7407bfe899f 100644 --- a/megatron/core/ssm/mamba_hybrid_layer_allocation.py +++ b/megatron/core/ssm/mamba_hybrid_layer_allocation.py @@ -1,20 +1,30 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import logging +from typing import Dict, List, Tuple if __name__ != "__main__": from megatron.core.utils import log_single_rank else: from typing import Any + import torch + def log_single_rank(logger: logging.Logger, *args: Any, rank: int = 0, **kwargs: Any): - print(*args[1:], **kwargs) + """Logs a message to the given rank.""" + if torch.distributed.is_initialized(): + if torch.distributed.get_rank() == rank: + logger.log(*args, **kwargs) + else: + logger.log(*args, **kwargs) logger = logging.getLogger(__name__) class Symbols: + """Symbols for different layer types.""" + MAMBA = "M" ATTENTION = "*" MLP = "-" @@ -87,6 +97,7 @@ def allocate_layers( target_mlp_ratio: float, override_pattern: str = None, ) -> list: + """Allocates layers according to the requested distribution of layer types.""" assert total_layers_count > 0 assert target_attention_ratio >= 0.0 and target_attention_ratio <= 1.0 assert target_mlp_ratio >= 0.0 and target_mlp_ratio <= 1.0 @@ -156,6 +167,22 @@ def allocate_layers( return layer_type_list +def get_layer_maps_from_layer_type_list( + layer_type_list: List[str], +) -> Tuple[Dict[int, int], Dict[int, int], Dict[int, int]]: + """ + Returns maps from global layer index to the corresponding layer index + for each layer type in [Attention, Mamba, MLP] given a layer type list. + """ + layer_types = [Symbols.ATTENTION, Symbols.MAMBA, Symbols.MLP] + layer_maps = {layer_type: {} for layer_type in layer_types} + for global_layer_idx, layer_type in enumerate(layer_type_list): + layer_map = layer_maps[layer_type] + local_layer_idx = len(layer_map) + layer_map[global_layer_idx] = local_layer_idx + return [layer_maps[layer_type] for layer_type in layer_types] + + if __name__ == "__main__": test_cases = [ # (10, 0.2, 0.0), @@ -187,5 +214,5 @@ def allocate_layers( (9, 0.0, 0.0, "MMMMMMMMM"), ] for t in test_cases: - print("") + logging.info("") allocate_layers(*t) diff --git a/megatron/core/ssm/mamba_layer.py b/megatron/core/ssm/mamba_layer.py index d83d518331c..69d5ef21c81 100644 --- a/megatron/core/ssm/mamba_layer.py +++ b/megatron/core/ssm/mamba_layer.py @@ -6,7 +6,7 @@ # LICENSE file in the root directory of this source tree. from dataclasses import dataclass, field -from typing import Dict, Optional, Union +from typing import Dict, Optional, Tuple, Union import torch from torch import Tensor @@ -82,6 +82,10 @@ def __init__( self.mamba_bda = build_module(submodules.mamba_bda) self.bias_dropout_add_exec_handler = torch.enable_grad + def mamba_state_shapes_per_request(self) -> Tuple[Tuple[int], Tuple[int]]: + """Returns the Mamba conv and ssm states shapes per request.""" + return self.mixer.mamba_state_shapes_per_request() + def forward( self, hidden_states: Tensor, @@ -127,10 +131,6 @@ def forward( return hidden_states - def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None): - """Allocate the inference cache.""" - return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype) - def sharded_state_dict( self, prefix: str = '', sharded_offsets: tuple = (), metadata: Optional[dict] = None ) -> ShardedStateDict: diff --git a/megatron/core/ssm/mamba_mixer.py b/megatron/core/ssm/mamba_mixer.py index 00067783ffa..b792f8a2f1f 100644 --- a/megatron/core/ssm/mamba_mixer.py +++ b/megatron/core/ssm/mamba_mixer.py @@ -29,7 +29,12 @@ make_sharded_tensors_for_checkpoint, sharded_state_dict_default, ) -from megatron.core.utils import deprecate_inference_params, log_single_rank +from megatron.core.utils import ( + check_mamba_sequence_packing_support, + deprecate_inference_params, + log_single_rank, + maybe_cat, +) from .mamba_context_parallel import MambaContextParallel @@ -40,6 +45,7 @@ try: from causal_conv1d import causal_conv1d_fn, causal_conv1d_update + from causal_conv1d.causal_conv1d_varlen import causal_conv1d_varlen_states except ImportError: causal_conv1d_fn = None causal_conv1d_update = None @@ -65,7 +71,6 @@ except ImportError: HAVE_EINOPS = False - logger = logging.getLogger(__name__) @@ -410,11 +415,11 @@ def forward( inference_context = deprecate_inference_params(inference_context, inference_params) - _, batch, dim = hidden_states.shape + in_inference_mode = inference_context is not None and not self.training + _, batch, dim = hidden_states.shape conv_state, ssm_state = None, None - in_inference_mode = inference_context is not None and not self.training if in_inference_mode: if inference_context.is_dynamic_batching(): return self.dynamic_inference(hidden_states, inference_context) @@ -442,27 +447,151 @@ def forward( return out, out_bias - def dynamic_inference( - self, hidden_states, inference_context: DynamicInferenceContext - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Runs inference computation for dynamic batching.""" - raise NotImplementedError(f"Dynamic inference is not supported.") + def dynamic_inference(self, hidden_states: torch.Tensor, context: DynamicInferenceContext): + """ + Executes dynamic inference by separating decode and prefill requests and + running them independently. Also runs the chunked prefill request independently + if it exists. + """ + sequence_packing_available, reason_for_no_sequence_packing = ( + check_mamba_sequence_packing_support() + ) + assert sequence_packing_available, reason_for_no_sequence_packing + + conv_state, ssm_state = context.mamba_states_cache(self.layer_number) + + # Fast path: decode-only + if context.is_decode_only(): + batch_indices = context.mamba_metadata.request_to_mamba_state_idx_cudagraph_only[ + : context.padded_active_token_count + ] + out, out_bias = self.decode( + hidden_states, conv_state, ssm_state, batch_indices=batch_indices + ) + return out, out_bias + + # Compute input projection before splitting into prefill and decode + # to ensure sequence parallel all-gather. + zxBCdt, _ = self.in_proj(hidden_states) + + # Compute split between decode and prefill. + seq_idx, cu_seqlens, return_varlen_states = self._get_varlen_generation_state(context) + active_query_lengths = context.request_query_lengths[ + context.paused_request_count : context.total_request_count + ] + batch_indices = context.mamba_metadata.request_to_mamba_state_idx + + # First request with query len > 1 is prefill-start. + first_prefill_token_idx = torch.nonzero(active_query_lengths > 1)[0].int() + + # Process decode requests if there are any. + if first_prefill_token_idx > 0: + zxBCdt_decode = zxBCdt[:first_prefill_token_idx] + batch_indices_decode = batch_indices[:first_prefill_token_idx] + y_decode = self.ssm_decode( + zxBCdt_decode.transpose(0, 1), conv_state, ssm_state, batch_indices_decode + ).transpose(0, 1) + else: + y_decode = None + + active_token_count = context.active_token_count + active_request_count = context.get_active_request_count() + padded_active_token_count = context.padded_active_token_count + + # Process the chunked prefill request if it exists. + if context.chunked_prefill_request_id != -1: + chunked_prefill_request_token_count = active_query_lengths[-1] + zxBCdt_chunked_prefill = zxBCdt[ + active_token_count - chunked_prefill_request_token_count : active_token_count + ] + batch_index_chunked_prefill = batch_indices[context.chunked_prefill_request_id] + + y_prefill_chunked = self.ssm_prefill( + zxBCdt_chunked_prefill, + conv_state=conv_state[batch_index_chunked_prefill].unsqueeze(0), + ssm_state=ssm_state[batch_index_chunked_prefill].unsqueeze(0), + is_chunked_prefill=True, + ) - def decode(self, hidden_states, conv_state, ssm_state) -> Tuple[torch.Tensor, torch.Tensor]: + # Remove the chunked prefill request from the request / token counts so + # the subsequent prefill computation ignores the chunked prefill request. + active_token_count -= chunked_prefill_request_token_count + active_request_count -= 1 + else: + y_prefill_chunked = None + + # Process non-chunked prefill requests if there are any. + if (remaining_prefill_tokens := active_token_count - first_prefill_token_idx) > 0: + zxBCdt_prefill = zxBCdt[first_prefill_token_idx:active_token_count] + cu_seqlens_prefill = F.pad( + cu_seqlens[first_prefill_token_idx + 1 : active_request_count + 1] + - first_prefill_token_idx, + (1, 0), + ) + seq_idx_prefill = ( + seq_idx[:, first_prefill_token_idx:active_token_count] - first_prefill_token_idx + ) + batch_indices_prefill = batch_indices[first_prefill_token_idx:active_request_count] + + y_prefill = self.ssm_prefill( + zxBCdt_prefill, + conv_state=conv_state, + ssm_state=ssm_state, + seq_idx=seq_idx_prefill, + cu_seqlens=cu_seqlens_prefill, + return_varlen_states=return_varlen_states, + batch_indices=batch_indices_prefill, + ) + else: + y_prefill = None + + # Assemble the final output by concatenating the decode output, + # non-chunked prefill output, and chunked prefill output together. + y_prefill = maybe_cat(y_prefill, y_prefill_chunked, required=True) + y = maybe_cat(y_decode, y_prefill, required=True) + + # Add padding tokens back if necessary. Note that we use the context active token count + # in case we modified the local count for chunked prefill above. + if (num_padding_tokens := padded_active_token_count - context.active_token_count) > 0: + y = torch.cat((y, y.new_zeros(num_padding_tokens, *y.shape[1:])), dim=0) + + # The output projection will perform the sequence parallel reduce-scatter if necessary. + out, out_bias = self.out_proj(y) + + return out, out_bias + + def decode( + self, hidden_states, conv_state, ssm_state, batch_indices: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: """Performs inference step for decoding.""" # assert self.ngroups_local_tp == 1, "Only support ngroups=1 for inference for now" - dtype = hidden_states.dtype - assert hidden_states.shape[0] == 1, "Only support decoding with 1 token at a time for now" + is_dynamic_batching = batch_indices is not None - # b d_model --> b p(2d) + if not is_dynamic_batching: + assert ( + hidden_states.shape[0] == 1 + ), "Only support decoding with 1 token at a time for now" + + # (1, b, d_model) -> (1, b, proj_dim) zxBCdt, _ = self.in_proj(hidden_states) + # Make batch size leading dimension since that is 1 + if is_dynamic_batching: + zxBCdt = zxBCdt.transpose(0, 1) + assert self.cp.cp_size == 1, "Context parallel not supported for Mamba inferenece decode" - y = self.ssm_decode(zxBCdt, conv_state=conv_state, ssm_state=ssm_state) + y = self.ssm_decode( + zxBCdt, conv_state=conv_state, ssm_state=ssm_state, batch_indices=batch_indices + ) + + # Restore sequence length as first dimension + if is_dynamic_batching: + y = y.transpose(0, 1) - # l b pd --> l b d + # y has shape (1, b, d_inner), which is what out_proj expects out, out_bias = self.out_proj(y) + return out, out_bias def ssm_training(self, zxBCdt: torch.Tensor) -> torch.Tensor: @@ -515,8 +644,34 @@ def ssm_prefill( zxBCdt: torch.Tensor, conv_state: Optional[torch.Tensor], ssm_state: Optional[torch.Tensor], + seq_idx: Optional[torch.Tensor] = None, + cu_seqlens: Optional[torch.Tensor] = None, + return_varlen_states: bool = False, + batch_indices: Optional[torch.Tensor] = None, + is_chunked_prefill: bool = False, ) -> torch.Tensor: - """Performs SSM computation for inference prefill step.""" + """ + Performs SSM computation for inference prefill step. + + Args: + zxBCdt: The input tensor of shape (l, b, d), which is a concatenation of + z, x, B, C, and dt projections. + conv_state: The convolution state tensor for inference. + ssm_state: The selective scan state tensor for inference. + seq_idx: A map from token index to request index for variable-length sequences. + cu_seqlens: Cumulative sequence lengths for variable-length sequences. + return_varlen_states: Whether to return variable-length states from the SSM kernel. + batch_indices: A map from batch id to position in the Mamba state tensors for + dynamic inference. + is_chunked_prefill: Whether the request is a chunked prefill request. + + Returns: + The output tensor of shape (l, b, d). + """ + is_dynamic_batching = seq_idx is not None + assert not ( + is_dynamic_batching and is_chunked_prefill + ), "Cannot use chunked prefill with dynamic batching" # transpose: l b pd --> b l pd zxBCdt = rearrange(zxBCdt, "l b d -> b l d").contiguous() @@ -534,29 +689,53 @@ def ssm_prefill( dim=-1, ) - # transpose: b l pd --> b pd l - xBC = rearrange(xBC, "b l d -> b d l").contiguous() - # Compute short convolution - if conv_state is not None: - # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv - # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. - conv_state.copy_(F.pad(xBC, (self.d_conv - xBC.shape[-1], 0))) # Update state (B D W) + if conv_state is not None and is_dynamic_batching: + # xBC should have shape (b l d) for causal_conv1d_varlen_states + assert batch_indices is not None + conv_state[batch_indices] = causal_conv1d_varlen_states( + xBC.squeeze(0), cu_seqlens, state_len=conv_state.shape[-1] + ) + + # Maintain channels-last memory layout to use seq_idx for causal_conv1d_fn + # See https://github.com/Dao-AILab/causal-conv1d/blob/69e6dadc28b169a4c49cb86b586f64ee90242c70/csrc/causal_conv1d.cpp#L174 # pylint: disable=line-too-long + xBC = xBC.transpose(1, 2) + elif is_chunked_prefill: + # Maintain channels-last memory layout to use initial_states for causal_conv1d_fn + # See https://github.com/Dao-AILab/causal-conv1d/blob/69e6dadc28b169a4c49cb86b586f64ee90242c70/csrc/causal_conv1d.cpp#L200 # pylint: disable=line-too-long + xBC = xBC.transpose(1, 2) + else: + # transpose: b l pd --> b pd l + xBC = rearrange(xBC, "b l d -> b d l").contiguous() + if conv_state is not None: + # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv + # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. + conv_state.copy_( + F.pad(xBC, (self.d_conv - xBC.shape[-1], 0)) + ) # Update state (B D W) seqlen = xBC.size(2) if causal_conv1d_fn is None: xBC = self.act(self.cp.conv1d(xBC)[..., :seqlen]) else: assert self.activation in ["silu", "swish"] + if is_chunked_prefill: + initial_conv_state = ( + conv_state[:, :, 1:].permute(0, 2, 1).contiguous().transpose(1, 2) + ) + else: + initial_conv_state = None xBC = causal_conv1d_fn( x=xBC, weight=rearrange(self.cp.get_conv1d_weight(), "d 1 w -> d w"), bias=self.cp.get_conv1d_bias(), activation=self.activation, + seq_idx=seq_idx, + initial_states=initial_conv_state, ) # transpose b pd l --> b l pd - xBC = rearrange(xBC, "b d l -> b l d").contiguous() + xBC = rearrange(xBC, "b d l -> b l d").contiguous() x, B, C = torch.split( xBC, @@ -583,6 +762,14 @@ def ssm_prefill( self.cp.cp_size == 1 or self.rmsnorm ), "Context parallel not supported for use_mem_eff_path==False and rmsnorm==False" + if is_chunked_prefill: + initial_ssm_state = ssm_state + else: + initial_ssm_state = None + + # Note that both `seq_idx` and `cu_seqlens` must be passed in + # for variable length generation. + # See https://github.com/state-spaces/mamba/blob/e0761ece1db07e0949dd88b4f4cd440420a19fd9/tests/test_generation.py#L97 # pylint: disable=line-too-long y = mamba_chunk_scan_combined( x, dt, @@ -599,11 +786,25 @@ def ssm_prefill( dt_bias=self.cp.get_dt_bias().float(), dt_softplus=True, return_final_states=ssm_state is not None, + seq_idx=seq_idx, + cu_seqlens=cu_seqlens, + return_varlen_states=return_varlen_states, + initial_states=initial_ssm_state, ) if ssm_state is not None: - y, last_state = y - ssm_state.copy_(last_state) + if return_varlen_states: + assert batch_indices is not None + + y, _, varlen_states = y + + # This has to be varlen_states, NOT last_state + # See reference implementation: + # https://github.com/state-spaces/mamba/blob/e0761ece1db07e0949dd88b4f4cd440420a19fd9/mamba_ssm/modules/mamba2.py#L267 # pylint: disable=line-too-long + ssm_state[batch_indices] = varlen_states + else: + y, last_state = y + ssm_state.copy_(last_state) y = rearrange(y, "b l h p -> l b (h p)").contiguous() y = self.cp.post_conv_ssm(y) @@ -616,14 +817,31 @@ def ssm_prefill( return y def ssm_decode( - self, zxBCdt: torch.Tensor, conv_state: torch.Tensor, ssm_state: torch.Tensor + self, + zxBCdt: torch.Tensor, + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + batch_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Performs SSM computation for inference decode step.""" - + """ + Performs SSM computation for inference decode step. + + Args: + zxBCdt: The input tensor of shape (l, b, d), which is a concatenation of + z, x, B, C, and dt projections. For decoding, l must be 1. + conv_state: The convolution state tensor for inference. + ssm_state: The selective scan state tensor for inference. + batch_indices: A map from batch id to position in the Mamba state tensors for + dynamic inference. + + Returns: + The output tensor of shape (l, b, d). + """ + seq_len, batch_size, _ = zxBCdt.shape dtype = zxBCdt.dtype - assert zxBCdt.shape[0] == 1, "Only support decoding with 1 token at a time for now" + assert seq_len == 1, "Only support decoding with 1 token at a time for now" - # l b d --> b d + # Remove sequence dimension zxBCdt = zxBCdt.squeeze(0) z, xBC, dt = torch.split( @@ -645,7 +863,7 @@ def ssm_decode( ) # (B D) if self.conv1d.bias is not None: xBC = xBC + self.conv1d.bias - xBC = self.act(xBC).to(dtype=dtype) + xBC = self.act(xBC).to(dtype=xBC.dtype) else: xBC = causal_conv1d_update( xBC, @@ -653,6 +871,7 @@ def ssm_decode( rearrange(self.conv1d.weight, "d 1 w -> d w"), self.conv1d.bias, self.activation, + conv_state_indices=batch_indices, ) x, B, C = torch.split( @@ -733,35 +952,61 @@ def ssm_decode( z=z if not self.rmsnorm else None, dt_bias=dt_bias, dt_softplus=True, + state_batch_indices=batch_indices, ) y = rearrange(y, "b h p -> b (h p)") if self.rmsnorm: y = self.norm(y, z) - # b (h p) -> l b (h p) + # Restore sequence dimension return y.unsqueeze(0) - def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None): - """ - allocate inference cache + def _get_varlen_generation_state( + self, inference_context: Optional[BaseInferenceContext] = None + ) -> Tuple[torch.Tensor, torch.Tensor, bool]: + """Constructs the variable length generation state for non-decode dynamic inference. + + The returned state includes the following: + `seq_idx` (Tensor): A map from token idx to request idx. + `cu_seqlens` (Tensor): The cumulative sequence lengths. + `return_varlen_states` (bool): Whether to return a varlen states tensor for + `mamba_chunk_scan_combined`. + + Returns empty state for training, static inference, or decode-only dynamic inference. + + Args: + inference_context (InferenceContext): The inference context. + + Returns: + A tuple of (`seq_idx`, `cu_seqlens`, `return_varlen_states`) """ - device = self.out_proj.weight.device - conv_dtype = self.conv1d.weight.dtype if dtype is None else dtype - conv_state = torch.zeros( - batch_size, self.conv1d.weight.shape[0], self.d_conv, device=device, dtype=conv_dtype - ) - ssm_dtype = self.in_proj.weight.dtype if dtype is None else dtype - # ssm_dtype = torch.float32 - ssm_state = torch.zeros( - batch_size, - self.nheads_local_tp, - self.headdim, - self.d_state, - device=device, - dtype=ssm_dtype, + + if ( + inference_context is None + or not inference_context.is_dynamic_batching() + or inference_context.is_decode_only() + ): + return None, None, False + + active_token_count = inference_context.active_token_count + seq_idx = ( + inference_context.token_to_request_idx[:active_token_count] + .clone() + .to(torch.int32) + .unsqueeze(0) ) - return conv_state, ssm_state + + # Get the list of cumulative sequence lengths for active requests. + cu_seqlens, _ = inference_context.cu_query_lengths() + + return seq_idx, cu_seqlens, True + + def mamba_state_shapes_per_request(self) -> Tuple[Tuple[int], Tuple[int]]: + """Returns the Mamba conv and ssm states shapes per request.""" + conv_states_shape = (self.conv1d.weight.shape[0], self.d_conv) + ssm_states_shape = (self.nheads_local_tp, self.headdim, self.d_state) + return (conv_states_shape, ssm_states_shape) def _get_states_from_cache(self, inference_context, batch_size, *, inference_params=None): """Initializes or retrieves the SSM state tensors from the cache. @@ -774,23 +1019,23 @@ def _get_states_from_cache(self, inference_context, batch_size, *, inference_par inference_context = deprecate_inference_params(inference_context, inference_params) assert inference_context is not None + assert inference_context.is_static_batching() assert self.layer_number is not None + if ( self.layer_number not in inference_context.key_value_memory_dict or batch_size != self.cached_batch_size ): + conv_state_shape, ssm_state_shape = self.mamba_state_shapes_per_request() conv_state = torch.zeros( batch_size, - self.conv1d.weight.shape[0], - self.d_conv, + *conv_state_shape, device=self.conv1d.weight.device, dtype=self.conv1d.weight.dtype, ) ssm_state = torch.zeros( batch_size, - self.nheads_local_tp, - self.headdim, - self.d_state, + *ssm_state_shape, device=self.in_proj.weight.device, dtype=self.in_proj.weight.dtype, ) @@ -798,7 +1043,6 @@ def _get_states_from_cache(self, inference_context, batch_size, *, inference_par self.cached_batch_size = batch_size else: conv_state, ssm_state = inference_context.key_value_memory_dict[self.layer_number] - # TODO: Remove reference to `inference_context.sequence_len_offset` for dynamic batching if inference_context.sequence_len_offset == 0: conv_state.zero_() ssm_state.zero_() diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index aab137b6430..895aef978e2 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -7,7 +7,7 @@ import torch import torch.nn.functional as F -from megatron.core.enums import Fp8Recipe +from megatron.core.enums import Fp4Recipe, Fp8Recipe from megatron.core.quantization.quant_config import RecipeConfig from megatron.core.transformer.enums import AttnBackend from megatron.core.transformer.pipeline_parallel_layer_layout import PipelineParallelLayerLayout @@ -382,10 +382,10 @@ class TransformerConfig(ModelParallelConfig): activation and weight tensors and e5m2 for all FP8 output activation gradient tensors.""" fp8_recipe: Optional[str] = "delayed" - """If set, enables the use of FP8 precision through Transformer Engine. There are 3 predefined + """If set, enables the use of FP8 precision through Transformer Engine. There are 5 predefined choices (1) 'tensorwise' uses per tensor current scaling recipe, (2) 'delayed' uses delayed scaling recipe, 3) 'mxfp8' for Blackwell architecture only, - 4) 'blockwise' for blockwise scaling recipe.""" + 4) 'blockwise' for blockwise scaling recipe, 5) 'custom' for custom quantization recipe.""" fp8_param: bool = False """If set, keep the parameters in fp8 precision to save memory. This option must be used @@ -394,6 +394,10 @@ class TransformerConfig(ModelParallelConfig): primarily the weights of GEMMs. The specific parameters that will be converted to fp8 are determined by TE.""" + fp8_quantizer_factory: Optional[str] = None + """Python import path to a callable quantizer factory, e.g., package.module.quantizer_factory. + Required when fp8_recipe is custom.""" + fp8_margin: int = 0 """Margin for the scaling factor computation.""" @@ -455,6 +459,10 @@ class TransformerConfig(ModelParallelConfig): together with fp4 mode (i.e., TransformerConfig.fp4 is not None). Note that not all parameters will be converted to fp4; for example, biases will remain unchanged.""" + fp4_quantizer_factory: Optional[str] = None + """Python import path to a callable quantizer factory, e.g., package.module.quantizer_factory. + Required when fp4_recipe is custom.""" + #################### # MoE related #################### @@ -909,6 +917,14 @@ def __post_init__(self): f"({max_bf16_layers_per_pipeline_stage})." ) + if self.fp8_recipe == Fp8Recipe.custom: + if not self.fp8_quantizer_factory: + raise ValueError( + "fp8_quantizer_factory must be provided when fp8_recipe is 'custom'. " + "Specify a Python import path (e.g., package.module.quantizer_factory) " + "via --fp8-quantizer-factory." + ) + if self.fp8_param and not self.fp8: raise ValueError("fp8_param must be used together with fp8 mode.") @@ -919,6 +935,14 @@ def __post_init__(self): if self.fp4 and self.fp8: raise ValueError("fp4 and fp8 cannot be used simultaneously. Please choose one.") + if self.fp4 and self.fp4_recipe == Fp4Recipe.custom: + if not self.fp4_quantizer_factory: + raise ValueError( + "fp4_quantizer_factory must be provided when fp4_recipe is 'custom'. " + "Specify a Python import path (e.g., package.module.quantizer_factory) " + "via --fp4-quantizer-factory." + ) + if self.apply_query_key_layer_scaling: self.attention_softmax_in_fp32 = True diff --git a/megatron/core/utils.py b/megatron/core/utils.py index 9947b8da683..8d6269e23a4 100644 --- a/megatron/core/utils.py +++ b/megatron/core/utils.py @@ -17,14 +17,16 @@ import time import traceback import warnings +from collections import defaultdict from contextlib import contextmanager, nullcontext from dataclasses import dataclass from datetime import datetime from functools import lru_cache, reduce, wraps from importlib.metadata import version from types import TracebackType -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple, Type, Union +import numpy import torch from megatron.core import config @@ -65,6 +67,8 @@ _torch_version = PkgVersion("0.0.0") if HAVE_PACKAGING else "0.0.0" _te_version = None _fa_version = None +_mamba_ssm_version = None +_causal_conv1d_version = None @contextmanager @@ -388,6 +392,79 @@ def is_fa_min_version(version, check_equality=True): return get_fa_version() > PkgVersion(version) +def get_mamba_version(): + """Get mamba version from __version__; if not available use pip's. Use caching.""" + if not HAVE_PACKAGING: + raise ImportError( + "packaging is not installed. Please install it with `pip install packaging`." + ) + + def get_mamba_version_str(): + import mamba_ssm + + if hasattr(mamba_ssm, "__version__"): + return str(mamba_ssm.__version__) + else: + return version("mamba_ssm") + + global _mamba_ssm_version + if _mamba_ssm_version is None: + _mamba_ssm_version = PkgVersion(get_mamba_version_str()) + return _mamba_ssm_version + + +def is_mamba_min_version(version, check_equality=True): + """Check if minimum version of `mamba_ssm` is installed.""" + if not HAVE_PACKAGING: + raise ImportError( + "packaging is not installed. Please install it with `pip install packaging`." + ) + if check_equality: + return get_mamba_version() >= PkgVersion(version) + return get_mamba_version() > PkgVersion(version) + + +def get_causal_conv1d_version(): + """Get causal_conv1d version from __version__; if not available use pip's. Use caching.""" + if not HAVE_PACKAGING: + raise ImportError( + "packaging is not installed. Please install it with `pip install packaging`." + ) + + def get_causal_conv1d_version_str(): + import causal_conv1d + + if hasattr(causal_conv1d, "__version__"): + return str(causal_conv1d.__version__) + else: + return version("causal_conv1d") + + global _causal_conv1d_version + if _causal_conv1d_version is None: + _causal_conv1d_version = PkgVersion(get_causal_conv1d_version_str()) + return _causal_conv1d_version + + +def is_causal_conv1d_min_version(version, check_equality=True): + """Check if minimum version of `causal_conv1d` is installed.""" + if not HAVE_PACKAGING: + raise ImportError( + "packaging is not installed. Please install it with `pip install packaging`." + ) + if check_equality: + return get_causal_conv1d_version() >= PkgVersion(version) + return get_causal_conv1d_version() > PkgVersion(version) + + +def check_mamba_sequence_packing_support() -> Tuple[bool, Optional[str]]: + """Checks whether `causal_conv1d` and `mamba_ssm` support sequence packing.""" + if not is_causal_conv1d_min_version("1.5.3.post1"): + return False, "causal_conv1d >= 1.5.3.post1 is required" + elif not is_mamba_min_version("2.2.6.post3"): + return False, "mamba_ssm >= 2.2.6.post3 is required" + return True, None + + def ensure_divisibility(numerator, denominator): """Ensure that numerator is divisible by the denominator.""" assert numerator % denominator == 0, "{} is not divisible by {}".format(numerator, denominator) @@ -2053,6 +2130,16 @@ def unwrap_model(model, module_instances=None): return unwrapped_model +def maybe_cat(a, b, dim=0, *, required=False): + """Concatenates `a` and `b` along `dim` if `a` and `b` exist.""" + xs = [t for t in (a, b) if t is not None] + if not xs: + if required: + raise ValueError("both tensors are None") + return None + return xs[0] if len(xs) == 1 else torch.cat(xs, dim=dim) + + def get_asyncio_loop(loop: asyncio.AbstractEventLoop | None = None) -> asyncio.AbstractEventLoop: """Creates an asyncio loop if necessary and then returns the current asyncio loop.""" if loop is None: @@ -2062,3 +2149,58 @@ def get_asyncio_loop(loop: asyncio.AbstractEventLoop | None = None) -> asyncio.A loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) return loop + + +_ASYNC_TASK_STATS = defaultdict(lambda: [0, 0.0]) # cnt, total_time + + +def trace_async_exceptions( + func: Optional[Callable[..., Coroutine]], *, verbose: bool = False +) -> Callable[..., Coroutine]: + """Decorator to be applied to every coroutine that runs in a separate task. + + This is needed because asyncio tasks do not propagate exceptions. + Coroutines running inside separate tasks will fail silently if not decorated. + + Passing in `verbose=True` will print additional lifetime logging information about the task. + Such functionality is relied on by some users, and can be enabled as shown below: + ``` + @trace_async_exceptions(verbose=True) + async def my_coroutine(...): + ... + ``` + """ + + def _decorate(fn): + if not asyncio.iscoroutinefunction(fn): + raise TypeError("trace_async_exceptions can only be used with async functions") + + @functools.wraps(fn) + async def wrapper(*args, **kwargs): + if verbose: + start = time.perf_counter() + try: + return await fn(*args, **kwargs) + except Exception as e: + logger.error(f"Exception in async function {fn.__name__}: {e}") + traceback.print_exc() + sys.exit(1) + finally: + if verbose: + elapsed = (time.perf_counter() - start) * 1000.0 + name = fn.__qualname__ + cnt, tot = _ASYNC_TASK_STATS[name] + _ASYNC_TASK_STATS[name] = [cnt + 1, tot + elapsed] + avg = _ASYNC_TASK_STATS[name][1] / _ASYNC_TASK_STATS[name][0] + + log10 = numpy.log10(max(cnt, 1)) + if numpy.isclose(log10, round(log10)): + logger.info( + f"{name} completed in {elapsed:.3f} ms, " + f"lifetime avg: {avg:.3f} ms, " + f"lifetime cnt: {cnt + 1}" + ) + + return wrapper + + return _decorate if func is None else _decorate(func) diff --git a/megatron/post_training/checkpointing.py b/megatron/post_training/checkpointing.py index cc26d5db2e3..aac59341e37 100644 --- a/megatron/post_training/checkpointing.py +++ b/megatron/post_training/checkpointing.py @@ -2,15 +2,13 @@ import logging from pathlib import Path -from typing import Dict, Optional, Tuple, Union +from typing import Optional, Tuple, Union import modelopt.torch.opt as mto -import torch import torch.nn as nn from modelopt.torch.opt.plugins import restore_sharded_modelopt_state from megatron.core import dist_checkpointing -from megatron.core.dist_checkpointing.strategies.common import COMMON_STATE_FNAME from megatron.core.utils import get_torch_version, is_torch_min_version from megatron.training import get_args from megatron.training.checkpointing import _load_base_checkpoint, load_checkpoint @@ -21,35 +19,36 @@ NEMO_WEIGHT_DIR_NAMES = {"model_weights": "model.", "weights": "module."} -def has_modelopt_state(checkpoint_path: str, ignore_kd_state: bool = False) -> bool: - """Check if modelopt_state folder exists inside the checkpoint path. +def has_modelopt_state(checkpoint_path: str) -> bool: + """Check if modelopt_state folder exists inside the checkpoint. Args: checkpoint_path: Path to the checkpoint directory - ignore_kd_state: If True, ignore the knowledge distillation state Returns: - True if modelopt_state folder exists when ignore_kd_state is False, - True if modelopt_state folder exists when ignore_kd_state is True and has only - distillation state, False otherwise + True if modelopt_state exists, False otherwise """ - load_dir, _ = get_sharded_load_dir(checkpoint_path) - if load_dir is None: - return False - modelopt_state_path = load_dir / "modelopt_state" - if not modelopt_state_path.is_dir(): - return False - elif ignore_kd_state: - return _has_only_kd_state(modelopt_state_path) - else: - return True - + args = get_args() -def _has_only_kd_state(modelopt_state_path: Path) -> bool: - modelopt_state = torch.load(modelopt_state_path / COMMON_STATE_FNAME, weights_only=False) - modes_dict = modelopt_state["modelopt_state_dict"] - if len(modes_dict) == 1 and modes_dict[0][0] == "kd_loss": - return True - return False + try: + if args.ckpt_format == "torch": + # Non-sharded + state_dict, _, _ = _load_base_checkpoint(checkpoint_path, rank0=False) + if state_dict is None: + return False + if "modelopt_state" not in state_dict: + return False + return True + else: + # Sharded + load_dir, _ = get_sharded_load_dir(checkpoint_path) + if load_dir is None: + return False + if not (load_dir / "modelopt_state").is_dir(): + return False + return True + except Exception as e: + print_rank_0(f"Failed to inspect checkpoint in {checkpoint_path}: {e}") + return False def get_sharded_load_dir(load_dir: str) -> Tuple[Union[Path, None], str]: @@ -89,41 +88,41 @@ def get_sharded_load_dir(load_dir: str) -> Tuple[Union[Path, None], str]: return sharded_load_dir, sharded_prefix -def load_modelopt_state(load_dir: Optional[str] = None, model: Optional[nn.Module] = None) -> Dict: +def load_modelopt_state(model: nn.Module, load_dir: Optional[str] = None) -> None: """Loading modelopt_state without loading the model. - If --use-dist-ckpt, we try to load from the sharded modelopt_state. This will not load the model - state_dict. Otherwise, if the checkpoint is not sharded, we load the base checkpoint (that - contains the model state as well) and extract the modelopt_state. + If distributed checkpointing in use, we try to load from the sharded modelopt_state. This will not + load the model state_dict. Otherwise, if the checkpoint is not sharded, we load the base checkpoint + (which contains the model state as well) and extract the modelopt_state. Args: + model: the model to load the modelopt_state into load_dir: optionally provide a different loading path - model: required when loading a sharded checkpoint """ args = get_args() + load_dir = load_dir or args.load - if load_dir is None: - load_dir = args.load - - if args.use_dist_ckpt: - assert model is not None, "`model` argument required when `args.use_dist_ckpt is True`" - sharded_load_dir, _ = get_sharded_load_dir(load_dir) - if sharded_load_dir is None: - print_rank_0("No sharded checkpoint found. Skipping loading modelopt_state.") - return {} - restore_sharded_modelopt_state([model], sharded_load_dir) - else: + if args.ckpt_format == "torch": + # Non-sharded print_rank_0(f"Loading ModelOpt state from base checkpoint ({load_dir})") try: state_dict, _, _ = _load_base_checkpoint(args.load, rank0=False) except Exception: print_rank_0("Failed to load base checkpoint via megatron _load_base_checkpoint!") + return if state_dict is None: print_rank_0("No checkpoint state_dict found. Skipping loading ModelOpt state.") - else: - modelopt_state = state_dict.get("modelopt_state", None) + return + modelopt_state = state_dict.get("modelopt_state", None) if modelopt_state is not None: mto.restore_from_modelopt_state(model, modelopt_state) + else: + # Sharded + sharded_load_dir, _ = get_sharded_load_dir(load_dir) + if sharded_load_dir is None: + print_rank_0("No sharded checkpoint found. Skipping loading modelopt_state.") + return + restore_sharded_modelopt_state([model], sharded_load_dir) def load_modelopt_checkpoint( diff --git a/megatron/rl/__init__.py b/megatron/rl/__init__.py index 035da465519..d3ae2fefd16 100644 --- a/megatron/rl/__init__.py +++ b/megatron/rl/__init__.py @@ -12,7 +12,6 @@ from pydantic import BaseModel, ConfigDict, Field from typing_extensions import Self, Type - def import_class(class_path: str) -> Type: """Import a class from a string path. @@ -76,43 +75,3 @@ class Request(BaseModel): """Generation Request.""" generation_args: GenericGenerationArgs = GenericGenerationArgs() - - -from collections import defaultdict - -_STATS = defaultdict(lambda: [0, 0.0]) # cnt, total_time - - -def trace_async_exceptions(fn: Callable[..., Coroutine]) -> Callable[..., Coroutine]: - """Decorator to be applied to every coroutine that runs in a separate task. - - This is needed because asyncio tasks do not propagate exceptions. - Coroutines running inside separate tasks will fail silently if not decorated. - """ - if not asyncio.iscoroutinefunction(fn): - raise TypeError("trace_async_exceptions can only be used with async functions") - - @functools.wraps(fn) - async def wrapper(*args, **kwargs): - start = time.perf_counter() - try: - return await fn(*args, **kwargs) - except Exception as e: - print(f"Exception in async function {fn.__name__}: {e}") - traceback.print_exc() - sys.exit(1) - finally: - elapsed = (time.perf_counter() - start) * 1000.0 - name = fn.__qualname__ - cnt, tot = _STATS[name] - _STATS[name] = [cnt + 1, tot + elapsed] - avg = _STATS[name][1] / _STATS[name][0] - import numpy as np - - log10 = np.log10(max(cnt, 1)) - if np.isclose(log10, round(log10)): - print( - f"{name} completed in {elapsed:.3f} ms, lifetime avg: {avg:.3f} ms, lifetime cnt: {cnt + 1}" - ) - - return wrapper diff --git a/megatron/rl/agent/api.py b/megatron/rl/agent/api.py index 3e16f74599f..fce7c3073ee 100644 --- a/megatron/rl/agent/api.py +++ b/megatron/rl/agent/api.py @@ -8,7 +8,7 @@ import numpy as np from pydantic import BaseModel -from ..__init__ import Request, TypeLookupable, trace_async_exceptions +from ..__init__ import Request, TypeLookupable from ..inference import ( ChatInferenceInterface, ChatInferenceRequest, @@ -18,6 +18,8 @@ ReturnsRaw, ) +from megatron.core.utils import trace_async_exceptions + class AgentBaseModel(BaseModel, extra='allow'): pass @@ -192,7 +194,7 @@ async def get_grouped_rollouts(self, request: GroupedRolloutRequest): ) submitted_groups = 0 - @trace_async_exceptions + @trace_async_exceptions(verbose=True) async def group_task(): nonlocal submitted_groups while request.num_groups == -1 or submitted_groups < request.num_groups: diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 5cb91d6c952..58613b364a6 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -183,7 +183,7 @@ async def base_generate(self, request: InferenceRequest): top_p=request.generation_args.top_p or 0.0, termination_id=self._coordinator.engine.controller.tokenizer.eod, return_log_probs=True, - skip_prompt_log_probs_for_dynamic_inference=True, + skip_prompt_log_probs=True, add_BOS=tokenizer.bos is not None, ) request_ids = [ diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 507c21e6883..494c82f7873 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1359,9 +1359,13 @@ def _add_transformer_engine_args(parser): dest='fp8') # per tensor current scaling recipe selection group.add_argument('--fp8-recipe', default='delayed', - choices=['tensorwise', 'delayed', 'mxfp8', 'blockwise'], + choices=['tensorwise', 'delayed', 'mxfp8', 'blockwise', 'custom'], help='Which fp8 recipe to use for FP8 tensors in the forward and backward pass', dest='fp8_recipe') + group.add_argument('--fp8-quantizer-factory', default=None, + help='Python import path to a callable quantizer factory, ' + 'e.g., package.module.quantizer_factory.', + dest='fp8_quantizer_factory') # delayed scaling only configs group.add_argument('--fp8-margin', type=int, default=0, help='Scaling margin for fp8', @@ -1398,9 +1402,13 @@ def _add_transformer_engine_args(parser): help='Which nvfp4 format scheme to use for FP4 tensors in the forward and backward pass', dest='fp4') group.add_argument('--fp4-recipe', default='nvfp4', - choices=['nvfp4'], + choices=['nvfp4', 'custom'], help='Which fp4 recipe to use for FP4 tensors in the forward and backward pass', dest='fp4_recipe') + group.add_argument('--fp4-quantizer-factory', default=None, + help='Python import path to a callable quantizer factory, ' + 'e.g., package.module.quantizer_factory.', + dest='fp4_quantizer_factory') group.add_argument('--fp4-param-gather', action='store_true', help='Keep the compute param in fp4 (do not use any other intermediate ' 'dtype) and perform the param all-gather in fp4.', diff --git a/megatron/training/tokenizer/sft_tokenizer.py b/megatron/training/tokenizer/sft_tokenizer.py index 4a941fc180b..f525352e892 100644 --- a/megatron/training/tokenizer/sft_tokenizer.py +++ b/megatron/training/tokenizer/sft_tokenizer.py @@ -170,6 +170,11 @@ def pad(self): """Pad token ID.""" return self._prompt_config.pad_token_id + @property + def bos(self): + """Beginning of sequence token ID.""" + return self._tokenizer.bos_token_id + @property def eod(self): """End of sentence token ID.""" diff --git a/megatron/training/training.py b/megatron/training/training.py index 06dad540fed..967397bec10 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -952,6 +952,18 @@ def get_model(model_provider_func, model_type=ModelType.encoder_or_decoder, wrap args = get_args() args.model_type = model_type + if has_nvidia_modelopt: + from megatron.post_training.checkpointing import has_modelopt_state + # [ModelOpt]: Check if the checkpoint is a ModelOpt checkpoint and + # set a flag to use our model provider if so. + if args.load is not None and has_modelopt_state(args.load): + print_rank_0(f'ModelOpt checkpoint detected') + args.modelopt_enabled = True + elif getattr(args, "export_kd_teacher_load", None): + # For distillation ckpts without ModelOpt state + args.modelopt_enabled = True + + # Build model. def build_model(): if ( @@ -1178,17 +1190,6 @@ def setup_model_and_optimizer( timers = get_timers() one_logger = get_one_logger() - if has_nvidia_modelopt: - from megatron.post_training.checkpointing import has_modelopt_state - # [ModelOpt]: Check if the checkpoint is a ModelOpt checkpoint and - # set a flag to use our model provider if so. - if args.load is not None and has_modelopt_state(args.load): - print_rank_0(f'ModelOpt checkpoint detected') - args.modelopt_enabled = True - elif getattr(args, "export_kd_teacher_load", None): - # For distillation ckpts without ModelOpt state - args.modelopt_enabled = True - model = get_model(model_provider_func, model_type) unwrapped_model = unwrap_model(model) diff --git a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py index 9a0758d3ba1..ae57db10e55 100644 --- a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py @@ -40,16 +40,17 @@ def test_inference_pipeline(golden_values_path: str, test_values_path: str) -> N # First warmup iteration is excluded from throughput statistics. throughput_sampled = median(output_current["throughput"][1:]) + throughput_golden = median(output_groundtruth["throughput"][1:]) # 10% is empirically observed to be within hardware variance. assert ( - throughput_sampled >= 0.9 * output_groundtruth["throughput"] - ), f"Throughput is slower than expected! Expected to be within 10% of ~{output_groundtruth['throughput']} tok/s but benchmarked {output_current['throughput']} tok/s" + throughput_sampled >= 0.9 * throughput_golden + ), f"Throughput is slower than expected! Expected to be within 10% of ~{throughput_golden} tok/s but benchmarked {output_current['throughput']} tok/s" # If throughput is significantly improved (> 20%), update golden values accordingly. assert ( - throughput_sampled < output_groundtruth["throughput"] * 1.2 - ), f"Throughput has been improved from expected ~{output_groundtruth['throughput']} tok/s to {output_current['throughput']} tok/s. Please update golden values in the functional tests." + throughput_sampled < throughput_golden * 1.2 + ), f"Throughput has been improved from expected ~{throughput_golden} tok/s to {output_current['throughput']} tok/s. Please update golden values in the functional tests." output_groundtruth.pop('throughput') diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json new file mode 100644 index 00000000000..3f7a0c2cac5 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84081, + "2": 10.8354, + "3": 10.83237, + "4": 10.81882, + "5": 10.84098, + "6": 10.87094, + "7": 10.83285, + "8": 10.8395, + "9": 10.84275, + "10": 10.80913, + "11": 10.85185, + "12": 10.84426, + "13": 10.86366, + "14": 10.86332, + "15": 10.80028, + "16": 10.79303, + "17": 10.7753, + "18": 10.80133, + "19": 10.79138, + "20": 10.70502, + "21": 10.68161, + "22": 10.56472, + "23": 10.70185, + "24": 10.58, + "25": 10.5355, + "26": 10.607, + "27": 10.59378, + "28": 10.56083, + "29": 10.57494, + "30": 10.35506, + "31": 10.12664, + "32": 10.46551, + "33": 10.45216, + "34": 10.22453, + "35": 10.27096, + "36": 10.22158, + "37": 10.33994, + "38": 10.18651, + "39": 10.39397, + "40": 10.07811, + "41": 10.13811, + "42": 10.20175, + "43": 9.83808, + "44": 9.94297, + "45": 9.82336, + "46": 9.82165, + "47": 10.13421, + "48": 9.84058, + "49": 9.52119, + "50": 9.90123 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1539.0, + "2": 1717.0, + "3": 1773.0, + "4": 1809.0, + "5": 1932.0, + "6": 1836.0, + "7": 1808.0, + "8": 1638.0, + "9": 1924.0, + "10": 1388.0, + "11": 1978.0, + "12": 1875.0, + "13": 1894.0, + "14": 1832.0, + "15": 1945.0, + "16": 1966.0, + "17": 1779.0, + "18": 1731.0, + "19": 1812.0, + "20": 1744.0, + "21": 1910.0, + "22": 1717.0, + "23": 2079.0, + "24": 1636.0, + "25": 1644.0, + "26": 1812.0, + "27": 1939.0, + "28": 1904.0, + "29": 2001.0, + "30": 2019.0, + "31": 1661.0, + "32": 1904.0, + "33": 2040.0, + "34": 1944.0, + "35": 1955.0, + "36": 1968.0, + "37": 2344.0, + "38": 2300.0, + "39": 2418.0, + "40": 2263.0, + "41": 2357.0, + "42": 2285.0, + "43": 1988.0, + "44": 2123.0, + "45": 2218.0, + "46": 2349.0, + "47": 2594.0, + "48": 2506.0, + "49": 2331.0, + "50": 2374.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4576453120.0, + "2": 4673069568.0, + "3": 4673069568.0, + "4": 4673069568.0, + "5": 4673069568.0, + "6": 4673069568.0, + "7": 4673069568.0, + "8": 4673069568.0, + "9": 4673069568.0, + "10": 4673069568.0, + "11": 4673069568.0, + "12": 4673069568.0, + "13": 4673069568.0, + "14": 4673069568.0, + "15": 4673069568.0, + "16": 4673069568.0, + "17": 4673069568.0, + "18": 4673069568.0, + "19": 4673069568.0, + "20": 4673069568.0, + "21": 4673069568.0, + "22": 4673069568.0, + "23": 4673069568.0, + "24": 4673069568.0, + "25": 4673069568.0, + "26": 4673069568.0, + "27": 4673069568.0, + "28": 4673069568.0, + "29": 4673069568.0, + "30": 4673069568.0, + "31": 4673069568.0, + "32": 4673069568.0, + "33": 4673069568.0, + "34": 4673069568.0, + "35": 4673069568.0, + "36": 4673069568.0, + "37": 4673069568.0, + "38": 4673069568.0, + "39": 4673069568.0, + "40": 4673069568.0, + "41": 4673069568.0, + "42": 4673069568.0, + "43": 4673069568.0, + "44": 4673069568.0, + "45": 4673069568.0, + "46": 4673069568.0, + "47": 4673069568.0, + "48": 4673069568.0, + "49": 4673069568.0, + "50": 4673069568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.18387, + "2": 0.12504, + "3": 0.09705, + "4": 0.09652, + "5": 0.09748, + "6": 0.09683, + "7": 0.09565, + "8": 0.09466, + "9": 0.09428, + "10": 0.09486, + "11": 0.09436, + "12": 0.09386, + "13": 0.09434, + "14": 0.09599, + "15": 0.09464, + "16": 0.0943, + "17": 0.09447, + "18": 0.09424, + "19": 0.0942, + "20": 0.09425, + "21": 0.09401, + "22": 0.09476, + "23": 0.09408, + "24": 0.09462, + "25": 0.09414, + "26": 0.09442, + "27": 0.0939, + "28": 0.09352, + "29": 0.09364, + "30": 0.09376, + "31": 0.09494, + "32": 0.09358, + "33": 0.09378, + "34": 0.09361, + "35": 0.09442, + "36": 0.09437, + "37": 0.09367, + "38": 0.0934, + "39": 0.09328, + "40": 0.09295, + "41": 0.09331, + "42": 0.09302, + "43": 0.09373, + "44": 0.09287, + "45": 0.09264, + "46": 0.10047, + "47": 0.09374, + "48": 0.09248, + "49": 0.09248, + "50": 0.09148 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json index a5a8b2f5f7e..dd30f7144c7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8401, - "5": 10.84032, - "10": 10.8134, - "15": 10.80276, - "20": 10.705, - "25": 10.53849, - "30": 10.35521, - "35": 10.27148, - "40": 10.08049, - "45": 9.8229, - "50": 9.90118 + "1": 10.84081, + "2": 10.8354, + "3": 10.83237, + "4": 10.81882, + "5": 10.84098, + "6": 10.87094, + "7": 10.83285, + "8": 10.8395, + "9": 10.84275, + "10": 10.80913, + "11": 10.85185, + "12": 10.84426, + "13": 10.86366, + "14": 10.86332, + "15": 10.80028, + "16": 10.79303, + "17": 10.7753, + "18": 10.80133, + "19": 10.79138, + "20": 10.70502, + "21": 10.68161, + "22": 10.56472, + "23": 10.70185, + "24": 10.58, + "25": 10.5355, + "26": 10.607, + "27": 10.59378, + "28": 10.56083, + "29": 10.57494, + "30": 10.35506, + "31": 10.12664, + "32": 10.46551, + "33": 10.45216, + "34": 10.22453, + "35": 10.27096, + "36": 10.22158, + "37": 10.33994, + "38": 10.18651, + "39": 10.39397, + "40": 10.07811, + "41": 10.13811, + "42": 10.20175, + "43": 9.83808, + "44": 9.94297, + "45": 9.82336, + "46": 9.82165, + "47": 10.13421, + "48": 9.84058, + "49": 9.52119, + "50": 9.90123 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1670.0, - "5": 1970.0, - "10": 1397.0, - "15": 1936.0, - "20": 1715.0, - "25": 1703.0, - "30": 1957.0, - "35": 1991.0, - "40": 2239.0, - "45": 2144.0, - "50": 2377.0 + "1": 1539.0, + "2": 1717.0, + "3": 1773.0, + "4": 1809.0, + "5": 1932.0, + "6": 1836.0, + "7": 1808.0, + "8": 1638.0, + "9": 1924.0, + "10": 1388.0, + "11": 1978.0, + "12": 1875.0, + "13": 1894.0, + "14": 1832.0, + "15": 1945.0, + "16": 1966.0, + "17": 1779.0, + "18": 1731.0, + "19": 1812.0, + "20": 1744.0, + "21": 1910.0, + "22": 1717.0, + "23": 2079.0, + "24": 1636.0, + "25": 1644.0, + "26": 1812.0, + "27": 1939.0, + "28": 1904.0, + "29": 2001.0, + "30": 2019.0, + "31": 1661.0, + "32": 1904.0, + "33": 2040.0, + "34": 1944.0, + "35": 1955.0, + "36": 1968.0, + "37": 2344.0, + "38": 2300.0, + "39": 2418.0, + "40": 2263.0, + "41": 2357.0, + "42": 2285.0, + "43": 1988.0, + "44": 2123.0, + "45": 2218.0, + "46": 2349.0, + "47": 2594.0, + "48": 2506.0, + "49": 2331.0, + "50": 2374.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 552238592.0, - "5": 552238592.0, - "10": 552238592.0, - "15": 552238592.0, - "20": 552238592.0, - "25": 552238592.0, - "30": 552238592.0, - "35": 552238592.0, - "40": 552238592.0, - "45": 552238592.0, - "50": 552238592.0 + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 4576563200.0, - "5": 4673179648.0, - "10": 4673179648.0, - "15": 4673179648.0, - "20": 4673179648.0, - "25": 4673179648.0, - "30": 4673179648.0, - "35": 4673179648.0, - "40": 4673179648.0, - "45": 4673179648.0, - "50": 4673179648.0 + "1": 4576453120.0, + "2": 4673069568.0, + "3": 4673069568.0, + "4": 4673069568.0, + "5": 4673069568.0, + "6": 4673069568.0, + "7": 4673069568.0, + "8": 4673069568.0, + "9": 4673069568.0, + "10": 4673069568.0, + "11": 4673069568.0, + "12": 4673069568.0, + "13": 4673069568.0, + "14": 4673069568.0, + "15": 4673069568.0, + "16": 4673069568.0, + "17": 4673069568.0, + "18": 4673069568.0, + "19": 4673069568.0, + "20": 4673069568.0, + "21": 4673069568.0, + "22": 4673069568.0, + "23": 4673069568.0, + "24": 4673069568.0, + "25": 4673069568.0, + "26": 4673069568.0, + "27": 4673069568.0, + "28": 4673069568.0, + "29": 4673069568.0, + "30": 4673069568.0, + "31": 4673069568.0, + "32": 4673069568.0, + "33": 4673069568.0, + "34": 4673069568.0, + "35": 4673069568.0, + "36": 4673069568.0, + "37": 4673069568.0, + "38": 4673069568.0, + "39": 4673069568.0, + "40": 4673069568.0, + "41": 4673069568.0, + "42": 4673069568.0, + "43": 4673069568.0, + "44": 4673069568.0, + "45": 4673069568.0, + "46": 4673069568.0, + "47": 4673069568.0, + "48": 4673069568.0, + "49": 4673069568.0, + "50": 4673069568.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.0358, - "5": 0.10098, - "10": 0.10007, - "15": 0.09866, - "20": 0.0986, - "25": 0.09963, - "30": 0.09768, - "35": 0.09735, - "40": 0.09791, - "45": 0.09757, - "50": 0.09769 + "1": 4.57734, + "2": 0.12447, + "3": 0.1105, + "4": 0.11652, + "5": 0.11171, + "6": 0.10268, + "7": 0.0964, + "8": 0.09397, + "9": 0.09475, + "10": 0.09372, + "11": 0.09325, + "12": 0.09309, + "13": 0.09305, + "14": 0.09354, + "15": 0.09324, + "16": 0.09342, + "17": 0.09327, + "18": 0.09347, + "19": 0.09283, + "20": 0.09308, + "21": 0.09266, + "22": 0.09487, + "23": 0.09318, + "24": 0.09338, + "25": 0.09306, + "26": 0.09374, + "27": 0.09386, + "28": 0.09412, + "29": 0.09395, + "30": 0.09393, + "31": 0.09439, + "32": 0.09481, + "33": 0.09338, + "34": 0.09466, + "35": 0.0936, + "36": 0.09463, + "37": 0.09316, + "38": 0.09572, + "39": 0.09295, + "40": 0.09592, + "41": 0.09322, + "42": 0.09468, + "43": 0.09488, + "44": 0.09323, + "45": 0.09265, + "46": 0.09574, + "47": 0.09267, + "48": 0.09592, + "49": 0.09356, + "50": 0.09502 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..d8d087006d9 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84454, + "13": 10.8632, + "14": 10.86353, + "15": 10.79985, + "16": 10.79265, + "17": 10.77473, + "18": 10.80156, + "19": 10.79155, + "20": 10.70512, + "21": 10.68174, + "22": 10.56547, + "23": 10.70145, + "24": 10.5789, + "25": 10.53597, + "26": 10.60745, + "27": 10.59418, + "28": 10.56116, + "29": 10.57573, + "30": 10.35474, + "31": 10.12618, + "32": 10.46569, + "33": 10.45235, + "34": 10.22491, + "35": 10.27088, + "36": 10.22167, + "37": 10.33935, + "38": 10.18639, + "39": 10.39432, + "40": 10.07794, + "41": 10.13875, + "42": 10.20184, + "43": 9.83819, + "44": 9.94273, + "45": 9.82299, + "46": 9.82187, + "47": 10.13444, + "48": 9.84097, + "49": 9.52094, + "50": 9.90107, + "51": 9.83459, + "52": 9.73231, + "53": 10.04881, + "54": 9.93895, + "55": 9.86297, + "56": 9.613, + "57": 9.46964, + "58": 9.81136, + "59": 9.57107, + "60": 9.48153, + "61": 9.67881, + "62": 9.96579, + "63": 9.35276, + "64": 9.75644, + "65": 8.93769, + "66": 9.68152, + "67": 9.35669, + "68": 9.76806, + "69": 9.7739, + "70": 9.71012, + "71": 9.60009, + "72": 9.56796, + "73": 9.47739, + "74": 8.93177, + "75": 9.40721, + "76": 9.06847, + "77": 10.0464, + "78": 9.70984, + "79": 9.35731, + "80": 9.38978, + "81": 9.4662, + "82": 9.68056, + "83": 9.29144, + "84": 9.40194, + "85": 9.59734, + "86": 9.06207, + "87": 9.57921, + "88": 9.73262, + "89": 9.58838, + "90": 9.80354, + "91": 9.31991, + "92": 9.35013, + "93": 9.06378, + "94": 8.81909, + "95": 9.50572, + "96": 9.51068, + "97": 9.29244, + "98": 9.65579, + "99": 8.87401, + "100": 9.38837 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1963.0, + "12": 1833.0, + "13": 1863.0, + "14": 1928.0, + "15": 1948.0, + "16": 1999.0, + "17": 1838.0, + "18": 1703.0, + "19": 1898.0, + "20": 1610.0, + "21": 1944.0, + "22": 1728.0, + "23": 2116.0, + "24": 1613.0, + "25": 1634.0, + "26": 1788.0, + "27": 2059.0, + "28": 2067.0, + "29": 1978.0, + "30": 1929.0, + "31": 1782.0, + "32": 1850.0, + "33": 2169.0, + "34": 1815.0, + "35": 2007.0, + "36": 2010.0, + "37": 2385.0, + "38": 2413.0, + "39": 2474.0, + "40": 2254.0, + "41": 2373.0, + "42": 2253.0, + "43": 1900.0, + "44": 2058.0, + "45": 2153.0, + "46": 2385.0, + "47": 2514.0, + "48": 2475.0, + "49": 2362.0, + "50": 2335.0, + "51": 2452.0, + "52": 2576.0, + "53": 2914.0, + "54": 2741.0, + "55": 2408.0, + "56": 2650.0, + "57": 2264.0, + "58": 2853.0, + "59": 2757.0, + "60": 2509.0, + "61": 3076.0, + "62": 2709.0, + "63": 2563.0, + "64": 3041.0, + "65": 2687.0, + "66": 3089.0, + "67": 2767.0, + "68": 2930.0, + "69": 2911.0, + "70": 3286.0, + "71": 3105.0, + "72": 2507.0, + "73": 3063.0, + "74": 2022.0, + "75": 2763.0, + "76": 3002.0, + "77": 3382.0, + "78": 3470.0, + "79": 3109.0, + "80": 3357.0, + "81": 3798.0, + "82": 3348.0, + "83": 2763.0, + "84": 3271.0, + "85": 3245.0, + "86": 2587.0, + "87": 3650.0, + "88": 3103.0, + "89": 3471.0, + "90": 3086.0, + "91": 3050.0, + "92": 3368.0, + "93": 2828.0, + "94": 3495.0, + "95": 3424.0, + "96": 3559.0, + "97": 3289.0, + "98": 3727.0, + "99": 3275.0, + "100": 3401.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0, + "51": 552128512.0, + "52": 552128512.0, + "53": 552128512.0, + "54": 552128512.0, + "55": 552128512.0, + "56": 552128512.0, + "57": 552128512.0, + "58": 552128512.0, + "59": 552128512.0, + "60": 552128512.0, + "61": 552128512.0, + "62": 552128512.0, + "63": 552128512.0, + "64": 552128512.0, + "65": 552128512.0, + "66": 552128512.0, + "67": 552128512.0, + "68": 552128512.0, + "69": 552128512.0, + "70": 552128512.0, + "71": 552128512.0, + "72": 552128512.0, + "73": 552128512.0, + "74": 552128512.0, + "75": 552128512.0, + "76": 552128512.0, + "77": 552128512.0, + "78": 552128512.0, + "79": 552128512.0, + "80": 552128512.0, + "81": 552128512.0, + "82": 552128512.0, + "83": 552128512.0, + "84": 552128512.0, + "85": 552128512.0, + "86": 552128512.0, + "87": 552128512.0, + "88": 552128512.0, + "89": 552128512.0, + "90": 552128512.0, + "91": 552128512.0, + "92": 552128512.0, + "93": 552128512.0, + "94": 552128512.0, + "95": 552128512.0, + "96": 552128512.0, + "97": 552128512.0, + "98": 552128512.0, + "99": 552128512.0, + "100": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2615097856.0, + "2": 2711714304.0, + "3": 2711714304.0, + "4": 2711714304.0, + "5": 2711714304.0, + "6": 2711714304.0, + "7": 2711714304.0, + "8": 2711714304.0, + "9": 2711714304.0, + "10": 2711714304.0, + "11": 2711714304.0, + "12": 2711714304.0, + "13": 2711714304.0, + "14": 2711714304.0, + "15": 2711714304.0, + "16": 2711714304.0, + "17": 2711714304.0, + "18": 2711714304.0, + "19": 2711714304.0, + "20": 2711714304.0, + "21": 2711714304.0, + "22": 2711714304.0, + "23": 2711714304.0, + "24": 2711714304.0, + "25": 2711714304.0, + "26": 2711714304.0, + "27": 2711714304.0, + "28": 2711714304.0, + "29": 2711714304.0, + "30": 2711714304.0, + "31": 2711714304.0, + "32": 2711714304.0, + "33": 2711714304.0, + "34": 2711714304.0, + "35": 2711714304.0, + "36": 2711714304.0, + "37": 2711714304.0, + "38": 2711714304.0, + "39": 2711714304.0, + "40": 2711714304.0, + "41": 2711714304.0, + "42": 2711714304.0, + "43": 2711714304.0, + "44": 2711714304.0, + "45": 2711714304.0, + "46": 2711714304.0, + "47": 2711714304.0, + "48": 2711714304.0, + "49": 2711714304.0, + "50": 2711714304.0, + "51": 2711714304.0, + "52": 2711714304.0, + "53": 2711714304.0, + "54": 2711714304.0, + "55": 2711714304.0, + "56": 2711714304.0, + "57": 2711714304.0, + "58": 2711714304.0, + "59": 2711714304.0, + "60": 2711714304.0, + "61": 2711714304.0, + "62": 2711714304.0, + "63": 2711714304.0, + "64": 2711714304.0, + "65": 2711714304.0, + "66": 2711714304.0, + "67": 2711714304.0, + "68": 2711714304.0, + "69": 2711714304.0, + "70": 2711714304.0, + "71": 2711714304.0, + "72": 2711714304.0, + "73": 2711714304.0, + "74": 2711714304.0, + "75": 2711714304.0, + "76": 2711714304.0, + "77": 2711714304.0, + "78": 2711714304.0, + "79": 2711714304.0, + "80": 2711714304.0, + "81": 2711714304.0, + "82": 2711714304.0, + "83": 2711714304.0, + "84": 2711714304.0, + "85": 2711714304.0, + "86": 2711714304.0, + "87": 2711714304.0, + "88": 2711714304.0, + "89": 2711714304.0, + "90": 2711714304.0, + "91": 2711714304.0, + "92": 2711714304.0, + "93": 2711714304.0, + "94": 2711714304.0, + "95": 2711714304.0, + "96": 2711714304.0, + "97": 2711714304.0, + "98": 2711714304.0, + "99": 2711714304.0, + "100": 2711714304.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.22026, + "2": 0.11508, + "3": 0.08784, + "4": 0.08853, + "5": 0.08761, + "6": 0.08872, + "7": 0.08738, + "8": 0.08924, + "9": 0.08542, + "10": 0.08527, + "11": 0.08496, + "12": 0.08437, + "13": 0.08471, + "14": 0.08501, + "15": 0.08468, + "16": 0.08444, + "17": 0.08421, + "18": 0.08402, + "19": 0.08444, + "20": 0.0841, + "21": 0.08395, + "22": 0.08442, + "23": 0.0846, + "24": 0.08394, + "25": 0.08371, + "26": 0.08416, + "27": 0.08354, + "28": 0.08445, + "29": 0.08405, + "30": 0.08414, + "31": 0.08444, + "32": 0.08369, + "33": 0.08356, + "34": 0.08435, + "35": 0.08405, + "36": 0.08358, + "37": 0.08349, + "38": 0.08439, + "39": 0.0837, + "40": 0.08444, + "41": 0.08399, + "42": 0.0835, + "43": 0.0843, + "44": 0.08389, + "45": 0.084, + "46": 0.08426, + "47": 0.0842, + "48": 0.08434, + "49": 0.08385, + "50": 0.08424, + "51": 0.09846, + "52": 0.08909, + "53": 0.08511, + "54": 0.0849, + "55": 0.1206, + "56": 0.0845, + "57": 0.08446, + "58": 0.08474, + "59": 0.08505, + "60": 0.08422, + "61": 0.08413, + "62": 0.0845, + "63": 0.08441, + "64": 0.08486, + "65": 0.08527, + "66": 0.08442, + "67": 0.08533, + "68": 0.08468, + "69": 0.08469, + "70": 0.08503, + "71": 0.08424, + "72": 0.085, + "73": 0.08469, + "74": 0.08484, + "75": 0.08396, + "76": 0.08437, + "77": 0.08458, + "78": 0.08553, + "79": 0.08492, + "80": 0.08459, + "81": 0.08431, + "82": 0.08515, + "83": 0.08407, + "84": 0.08429, + "85": 0.08413, + "86": 0.08499, + "87": 0.08442, + "88": 0.08461, + "89": 0.08478, + "90": 0.08469, + "91": 0.08469, + "92": 0.08478, + "93": 0.08453, + "94": 0.0842, + "95": 0.08391, + "96": 0.08383, + "97": 0.08459, + "98": 0.08469, + "99": 0.085, + "100": 0.08518 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json index 66e69ad5f5e..9dad9972e22 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8401, - "5": 10.84032, - "10": 10.8134, - "15": 10.80276, - "20": 10.705, - "25": 10.53849, - "30": 10.35521, - "35": 10.27148, - "40": 10.08049, - "45": 9.8229, - "50": 9.90118, - "55": 9.86424, - "60": 9.4803, - "65": 8.93743, - "70": 9.71026, - "75": 9.4088, - "80": 9.39078, - "85": 9.59741, - "90": 9.80389, - "95": 9.50562, - "100": 9.38808 + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84454, + "13": 10.8632, + "14": 10.86353, + "15": 10.79985, + "16": 10.79265, + "17": 10.77473, + "18": 10.80156, + "19": 10.79155, + "20": 10.70512, + "21": 10.68174, + "22": 10.56547, + "23": 10.70145, + "24": 10.5789, + "25": 10.53597, + "26": 10.60745, + "27": 10.59418, + "28": 10.56116, + "29": 10.57573, + "30": 10.35474, + "31": 10.12618, + "32": 10.46569, + "33": 10.45235, + "34": 10.22491, + "35": 10.27088, + "36": 10.22167, + "37": 10.33935, + "38": 10.18639, + "39": 10.39432, + "40": 10.07794, + "41": 10.13875, + "42": 10.20184, + "43": 9.83819, + "44": 9.94273, + "45": 9.82299, + "46": 9.82187, + "47": 10.13444, + "48": 9.84097, + "49": 9.52094, + "50": 9.90107, + "51": 9.83459, + "52": 9.73231, + "53": 10.04881, + "54": 9.93895, + "55": 9.86297, + "56": 9.613, + "57": 9.46964, + "58": 9.81136, + "59": 9.57107, + "60": 9.48153, + "61": 9.67881, + "62": 9.96579, + "63": 9.35276, + "64": 9.75644, + "65": 8.93769, + "66": 9.68152, + "67": 9.35669, + "68": 9.76806, + "69": 9.7739, + "70": 9.71012, + "71": 9.60009, + "72": 9.56796, + "73": 9.47739, + "74": 8.93177, + "75": 9.40721, + "76": 9.06847, + "77": 10.0464, + "78": 9.70984, + "79": 9.35731, + "80": 9.38978, + "81": 9.4662, + "82": 9.68056, + "83": 9.29144, + "84": 9.40194, + "85": 9.59734, + "86": 9.06207, + "87": 9.57921, + "88": 9.73262, + "89": 9.58838, + "90": 9.80354, + "91": 9.31991, + "92": 9.35013, + "93": 9.06378, + "94": 8.81909, + "95": 9.50572, + "96": 9.51068, + "97": 9.29244, + "98": 9.65579, + "99": 8.87401, + "100": 9.38837 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1670.0, - "5": 1970.0, - "10": 1397.0, - "15": 1936.0, - "20": 1715.0, - "25": 1703.0, - "30": 1957.0, - "35": 1991.0, - "40": 2239.0, - "45": 2144.0, - "50": 2377.0, - "55": 2436.0, - "60": 2445.0, - "65": 2645.0, - "70": 3337.0, - "75": 2726.0, - "80": 3356.0, - "85": 3336.0, - "90": 3044.0, - "95": 3484.0, - "100": 3467.0 + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1963.0, + "12": 1833.0, + "13": 1863.0, + "14": 1928.0, + "15": 1948.0, + "16": 1999.0, + "17": 1838.0, + "18": 1703.0, + "19": 1898.0, + "20": 1610.0, + "21": 1944.0, + "22": 1728.0, + "23": 2116.0, + "24": 1613.0, + "25": 1634.0, + "26": 1788.0, + "27": 2059.0, + "28": 2067.0, + "29": 1978.0, + "30": 1929.0, + "31": 1782.0, + "32": 1850.0, + "33": 2169.0, + "34": 1815.0, + "35": 2007.0, + "36": 2010.0, + "37": 2385.0, + "38": 2413.0, + "39": 2474.0, + "40": 2254.0, + "41": 2373.0, + "42": 2253.0, + "43": 1900.0, + "44": 2058.0, + "45": 2153.0, + "46": 2385.0, + "47": 2514.0, + "48": 2475.0, + "49": 2362.0, + "50": 2335.0, + "51": 2452.0, + "52": 2576.0, + "53": 2914.0, + "54": 2741.0, + "55": 2408.0, + "56": 2650.0, + "57": 2264.0, + "58": 2853.0, + "59": 2757.0, + "60": 2509.0, + "61": 3076.0, + "62": 2709.0, + "63": 2563.0, + "64": 3041.0, + "65": 2687.0, + "66": 3089.0, + "67": 2767.0, + "68": 2930.0, + "69": 2911.0, + "70": 3286.0, + "71": 3105.0, + "72": 2507.0, + "73": 3063.0, + "74": 2022.0, + "75": 2763.0, + "76": 3002.0, + "77": 3382.0, + "78": 3470.0, + "79": 3109.0, + "80": 3357.0, + "81": 3798.0, + "82": 3348.0, + "83": 2763.0, + "84": 3271.0, + "85": 3245.0, + "86": 2587.0, + "87": 3650.0, + "88": 3103.0, + "89": 3471.0, + "90": 3086.0, + "91": 3050.0, + "92": 3368.0, + "93": 2828.0, + "94": 3495.0, + "95": 3424.0, + "96": 3559.0, + "97": 3289.0, + "98": 3727.0, + "99": 3275.0, + "100": 3401.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 552238592.0, - "5": 552238592.0, - "10": 552238592.0, - "15": 552238592.0, - "20": 552238592.0, - "25": 552238592.0, - "30": 552238592.0, - "35": 552238592.0, - "40": 552238592.0, - "45": 552238592.0, - "50": 552238592.0, - "55": 552238592.0, - "60": 552238592.0, - "65": 552238592.0, - "70": 552238592.0, - "75": 552238592.0, - "80": 552238592.0, - "85": 552238592.0, - "90": 552238592.0, - "95": 552238592.0, - "100": 552238592.0 + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0, + "51": 552128512.0, + "52": 552128512.0, + "53": 552128512.0, + "54": 552128512.0, + "55": 552128512.0, + "56": 552128512.0, + "57": 552128512.0, + "58": 552128512.0, + "59": 552128512.0, + "60": 552128512.0, + "61": 552128512.0, + "62": 552128512.0, + "63": 552128512.0, + "64": 552128512.0, + "65": 552128512.0, + "66": 552128512.0, + "67": 552128512.0, + "68": 552128512.0, + "69": 552128512.0, + "70": 552128512.0, + "71": 552128512.0, + "72": 552128512.0, + "73": 552128512.0, + "74": 552128512.0, + "75": 552128512.0, + "76": 552128512.0, + "77": 552128512.0, + "78": 552128512.0, + "79": 552128512.0, + "80": 552128512.0, + "81": 552128512.0, + "82": 552128512.0, + "83": 552128512.0, + "84": 552128512.0, + "85": 552128512.0, + "86": 552128512.0, + "87": 552128512.0, + "88": 552128512.0, + "89": 552128512.0, + "90": 552128512.0, + "91": 552128512.0, + "92": 552128512.0, + "93": 552128512.0, + "94": 552128512.0, + "95": 552128512.0, + "96": 552128512.0, + "97": 552128512.0, + "98": 552128512.0, + "99": 552128512.0, + "100": 552128512.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 4576563200.0, - "5": 4673179648.0, - "10": 4673179648.0, - "15": 4673179648.0, - "20": 4673179648.0, - "25": 4673179648.0, - "30": 4673179648.0, - "35": 4673179648.0, - "40": 4673179648.0, - "45": 4673179648.0, - "50": 4673179648.0, - "55": 4673179648.0, - "60": 4673179648.0, - "65": 4673179648.0, - "70": 4673179648.0, - "75": 4673179648.0, - "80": 4673179648.0, - "85": 4673179648.0, - "90": 4673179648.0, - "95": 4673179648.0, - "100": 4673179648.0 + "1": 2615097856.0, + "2": 2711714304.0, + "3": 2711714304.0, + "4": 2711714304.0, + "5": 2711714304.0, + "6": 2711714304.0, + "7": 2711714304.0, + "8": 2711714304.0, + "9": 2711714304.0, + "10": 2711714304.0, + "11": 2711714304.0, + "12": 2711714304.0, + "13": 2711714304.0, + "14": 2711714304.0, + "15": 2711714304.0, + "16": 2711714304.0, + "17": 2711714304.0, + "18": 2711714304.0, + "19": 2711714304.0, + "20": 2711714304.0, + "21": 2711714304.0, + "22": 2711714304.0, + "23": 2711714304.0, + "24": 2711714304.0, + "25": 2711714304.0, + "26": 2711714304.0, + "27": 2711714304.0, + "28": 2711714304.0, + "29": 2711714304.0, + "30": 2711714304.0, + "31": 2711714304.0, + "32": 2711714304.0, + "33": 2711714304.0, + "34": 2711714304.0, + "35": 2711714304.0, + "36": 2711714304.0, + "37": 2711714304.0, + "38": 2711714304.0, + "39": 2711714304.0, + "40": 2711714304.0, + "41": 2711714304.0, + "42": 2711714304.0, + "43": 2711714304.0, + "44": 2711714304.0, + "45": 2711714304.0, + "46": 2711714304.0, + "47": 2711714304.0, + "48": 2711714304.0, + "49": 2711714304.0, + "50": 2711714304.0, + "51": 2711714304.0, + "52": 2711714304.0, + "53": 2711714304.0, + "54": 2711714304.0, + "55": 2711714304.0, + "56": 2711714304.0, + "57": 2711714304.0, + "58": 2711714304.0, + "59": 2711714304.0, + "60": 2711714304.0, + "61": 2711714304.0, + "62": 2711714304.0, + "63": 2711714304.0, + "64": 2711714304.0, + "65": 2711714304.0, + "66": 2711714304.0, + "67": 2711714304.0, + "68": 2711714304.0, + "69": 2711714304.0, + "70": 2711714304.0, + "71": 2711714304.0, + "72": 2711714304.0, + "73": 2711714304.0, + "74": 2711714304.0, + "75": 2711714304.0, + "76": 2711714304.0, + "77": 2711714304.0, + "78": 2711714304.0, + "79": 2711714304.0, + "80": 2711714304.0, + "81": 2711714304.0, + "82": 2711714304.0, + "83": 2711714304.0, + "84": 2711714304.0, + "85": 2711714304.0, + "86": 2711714304.0, + "87": 2711714304.0, + "88": 2711714304.0, + "89": 2711714304.0, + "90": 2711714304.0, + "91": 2711714304.0, + "92": 2711714304.0, + "93": 2711714304.0, + "94": 2711714304.0, + "95": 2711714304.0, + "96": 2711714304.0, + "97": 2711714304.0, + "98": 2711714304.0, + "99": 2711714304.0, + "100": 2711714304.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.31173, - "5": 0.10511, - "10": 0.10337, - "15": 0.10143, - "20": 0.10201, - "25": 0.10088, - "30": 0.10154, - "35": 0.1006, - "40": 0.10106, - "45": 0.10038, - "50": 0.10112, - "55": 0.10263, - "60": 0.0993, - "65": 0.09963, - "70": 0.09998, - "75": 0.09811, - "80": 0.09929, - "85": 0.09785, - "90": 0.09909, - "95": 0.09864, - "100": 0.09877 + "1": 4.10688, + "2": 0.11397, + "3": 0.08797, + "4": 0.08663, + "5": 0.08687, + "6": 0.08702, + "7": 0.08653, + "8": 0.08674, + "9": 0.08696, + "10": 0.08678, + "11": 0.08635, + "12": 0.08637, + "13": 0.08738, + "14": 0.08674, + "15": 0.08706, + "16": 0.08684, + "17": 0.08681, + "18": 0.08601, + "19": 0.08591, + "20": 0.08645, + "21": 0.08634, + "22": 0.08598, + "23": 0.08618, + "24": 0.08622, + "25": 0.08632, + "26": 0.08621, + "27": 0.08644, + "28": 0.08581, + "29": 0.08622, + "30": 0.08652, + "31": 0.08679, + "32": 0.08526, + "33": 0.08525, + "34": 0.08525, + "35": 0.08519, + "36": 0.08535, + "37": 0.08568, + "38": 0.0852, + "39": 0.08521, + "40": 0.08523, + "41": 0.08535, + "42": 0.08486, + "43": 0.08614, + "44": 0.08491, + "45": 0.08554, + "46": 0.08508, + "47": 0.08524, + "48": 0.08608, + "49": 0.08565, + "50": 0.08559, + "51": 0.10342, + "52": 0.09048, + "53": 0.08707, + "54": 0.08719, + "55": 0.08631, + "56": 0.11667, + "57": 0.08592, + "58": 0.08517, + "59": 0.08612, + "60": 0.08514, + "61": 0.0855, + "62": 0.08527, + "63": 0.08586, + "64": 0.08556, + "65": 0.08633, + "66": 0.08532, + "67": 0.08593, + "68": 0.08563, + "69": 0.08537, + "70": 0.08538, + "71": 0.08507, + "72": 0.08593, + "73": 0.08623, + "74": 0.08561, + "75": 0.08536, + "76": 0.08551, + "77": 0.08526, + "78": 0.0859, + "79": 0.08518, + "80": 0.08601, + "81": 0.08574, + "82": 0.08618, + "83": 0.08532, + "84": 0.08505, + "85": 0.08545, + "86": 0.08554, + "87": 0.08542, + "88": 0.08575, + "89": 0.0861, + "90": 0.08516, + "91": 0.08552, + "92": 0.08581, + "93": 0.08558, + "94": 0.08577, + "95": 0.08708, + "96": 0.08574, + "97": 0.08543, + "98": 0.0855, + "99": 0.08537, + "100": 0.08541 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json new file mode 100644 index 00000000000..c6d7c9c90fa --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84449, + "13": 10.86322, + "14": 10.86353, + "15": 10.79981, + "16": 10.79262, + "17": 10.77477, + "18": 10.80157, + "19": 10.79148, + "20": 10.70508, + "21": 10.68176, + "22": 10.56548, + "23": 10.70147, + "24": 10.57889, + "25": 10.53597, + "26": 10.60742, + "27": 10.59423, + "28": 10.56119, + "29": 10.57569, + "30": 10.35474, + "31": 10.12616, + "32": 10.46566, + "33": 10.45233, + "34": 10.22493, + "35": 10.27091, + "36": 10.22168, + "37": 10.33936, + "38": 10.18641, + "39": 10.39431, + "40": 10.07792, + "41": 10.13872, + "42": 10.20182, + "43": 9.83818, + "44": 9.94274, + "45": 9.82303, + "46": 9.82185, + "47": 10.13443, + "48": 9.84098, + "49": 9.52095, + "50": 9.90109, + "51": 9.83457, + "52": 9.73232, + "53": 10.0488, + "54": 9.93895, + "55": 9.863, + "56": 9.613, + "57": 9.46966, + "58": 9.81135, + "59": 9.57107, + "60": 9.48155, + "61": 9.6788, + "62": 9.96581, + "63": 9.35273, + "64": 9.75648, + "65": 8.93771, + "66": 9.68153, + "67": 9.35671, + "68": 9.76807, + "69": 9.7739, + "70": 9.71016, + "71": 9.60009, + "72": 9.56793, + "73": 9.4774, + "74": 8.93177, + "75": 9.4072, + "76": 9.06849, + "77": 10.0464, + "78": 9.70988, + "79": 9.35733, + "80": 9.38975, + "81": 9.4662, + "82": 9.68058, + "83": 9.2914, + "84": 9.40191, + "85": 9.59735, + "86": 9.06209, + "87": 9.57922, + "88": 9.73259, + "89": 9.58836, + "90": 9.80354, + "91": 9.31991, + "92": 9.35011, + "93": 9.06382, + "94": 8.81909, + "95": 9.50568, + "96": 9.51071, + "97": 9.29241, + "98": 9.65578, + "99": 8.87401, + "100": 9.38833 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1966.0, + "12": 1767.0, + "13": 1876.0, + "14": 1881.0, + "15": 1928.0, + "16": 1875.0, + "17": 1790.0, + "18": 1736.0, + "19": 1804.0, + "20": 1690.0, + "21": 2008.0, + "22": 1765.0, + "23": 2073.0, + "24": 1618.0, + "25": 1720.0, + "26": 1807.0, + "27": 1861.0, + "28": 2026.0, + "29": 1982.0, + "30": 1981.0, + "31": 1688.0, + "32": 1913.0, + "33": 2123.0, + "34": 1893.0, + "35": 2007.0, + "36": 1987.0, + "37": 2334.0, + "38": 2223.0, + "39": 2417.0, + "40": 2370.0, + "41": 2352.0, + "42": 2269.0, + "43": 1967.0, + "44": 2183.0, + "45": 2150.0, + "46": 2350.0, + "47": 2555.0, + "48": 2463.0, + "49": 2326.0, + "50": 2270.0, + "51": 2508.0, + "52": 2495.0, + "53": 2856.0, + "54": 2692.0, + "55": 2482.0, + "56": 2614.0, + "57": 2283.0, + "58": 2894.0, + "59": 2659.0, + "60": 2561.0, + "61": 3006.0, + "62": 2671.0, + "63": 2488.0, + "64": 3092.0, + "65": 2622.0, + "66": 3108.0, + "67": 2741.0, + "68": 2942.0, + "69": 2983.0, + "70": 3347.0, + "71": 3034.0, + "72": 2438.0, + "73": 3075.0, + "74": 1931.0, + "75": 2722.0, + "76": 2960.0, + "77": 3387.0, + "78": 3268.0, + "79": 3079.0, + "80": 3404.0, + "81": 3674.0, + "82": 3192.0, + "83": 2791.0, + "84": 3224.0, + "85": 3237.0, + "86": 2646.0, + "87": 3840.0, + "88": 3114.0, + "89": 3410.0, + "90": 3184.0, + "91": 3073.0, + "92": 3396.0, + "93": 2711.0, + "94": 3530.0, + "95": 3387.0, + "96": 3530.0, + "97": 3277.0, + "98": 3775.0, + "99": 3421.0, + "100": 3350.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1234585088.0, + "2": 1234585088.0, + "3": 1234585088.0, + "4": 1234585088.0, + "5": 1234585088.0, + "6": 1234585088.0, + "7": 1234585088.0, + "8": 1234585088.0, + "9": 1234585088.0, + "10": 1234585088.0, + "11": 1234585088.0, + "12": 1234585088.0, + "13": 1234585088.0, + "14": 1234585088.0, + "15": 1234585088.0, + "16": 1234585088.0, + "17": 1234585088.0, + "18": 1234585088.0, + "19": 1234585088.0, + "20": 1234585088.0, + "21": 1234585088.0, + "22": 1234585088.0, + "23": 1234585088.0, + "24": 1234585088.0, + "25": 1234585088.0, + "26": 1234585088.0, + "27": 1234585088.0, + "28": 1234585088.0, + "29": 1234585088.0, + "30": 1234585088.0, + "31": 1234585088.0, + "32": 1234585088.0, + "33": 1234585088.0, + "34": 1234585088.0, + "35": 1234585088.0, + "36": 1234585088.0, + "37": 1234585088.0, + "38": 1234585088.0, + "39": 1234585088.0, + "40": 1234585088.0, + "41": 1234585088.0, + "42": 1234585088.0, + "43": 1234585088.0, + "44": 1234585088.0, + "45": 1234585088.0, + "46": 1234585088.0, + "47": 1234585088.0, + "48": 1234585088.0, + "49": 1234585088.0, + "50": 1234585088.0, + "51": 1234585088.0, + "52": 1234585088.0, + "53": 1234585088.0, + "54": 1234585088.0, + "55": 1234585088.0, + "56": 1234585088.0, + "57": 1234585088.0, + "58": 1234585088.0, + "59": 1234585088.0, + "60": 1234585088.0, + "61": 1234585088.0, + "62": 1234585088.0, + "63": 1234585088.0, + "64": 1234585088.0, + "65": 1234585088.0, + "66": 1234585088.0, + "67": 1234585088.0, + "68": 1234585088.0, + "69": 1234585088.0, + "70": 1234585088.0, + "71": 1234585088.0, + "72": 1234585088.0, + "73": 1234585088.0, + "74": 1234585088.0, + "75": 1234585088.0, + "76": 1234585088.0, + "77": 1234585088.0, + "78": 1234585088.0, + "79": 1234585088.0, + "80": 1234585088.0, + "81": 1234585088.0, + "82": 1234585088.0, + "83": 1234585088.0, + "84": 1234585088.0, + "85": 1234585088.0, + "86": 1234585088.0, + "87": 1234585088.0, + "88": 1234585088.0, + "89": 1234585088.0, + "90": 1234585088.0, + "91": 1234585088.0, + "92": 1234585088.0, + "93": 1234585088.0, + "94": 1234585088.0, + "95": 1234585088.0, + "96": 1234585088.0, + "97": 1234585088.0, + "98": 1234585088.0, + "99": 1234585088.0, + "100": 1234585088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1984492544.0, + "2": 2536167424.0, + "3": 2536167424.0, + "4": 2536167424.0, + "5": 2536167424.0, + "6": 2536167424.0, + "7": 2536167424.0, + "8": 2536167424.0, + "9": 2536167424.0, + "10": 2536167424.0, + "11": 2536167424.0, + "12": 2536167424.0, + "13": 2536167424.0, + "14": 2536167424.0, + "15": 2536167424.0, + "16": 2536167424.0, + "17": 2536167424.0, + "18": 2536167424.0, + "19": 2536167424.0, + "20": 2536167424.0, + "21": 2536167424.0, + "22": 2536167424.0, + "23": 2536167424.0, + "24": 2536167424.0, + "25": 2536167424.0, + "26": 2536167424.0, + "27": 2536167424.0, + "28": 2536167424.0, + "29": 2536167424.0, + "30": 2536167424.0, + "31": 2536167424.0, + "32": 2536167424.0, + "33": 2536167424.0, + "34": 2536167424.0, + "35": 2536167424.0, + "36": 2536167424.0, + "37": 2536167424.0, + "38": 2536167424.0, + "39": 2536167424.0, + "40": 2536167424.0, + "41": 2536167424.0, + "42": 2536167424.0, + "43": 2536167424.0, + "44": 2536167424.0, + "45": 2536167424.0, + "46": 2536167424.0, + "47": 2536167424.0, + "48": 2536167424.0, + "49": 2536167424.0, + "50": 2536167424.0, + "51": 2536167424.0, + "52": 2536167424.0, + "53": 2536167424.0, + "54": 2536167424.0, + "55": 2536167424.0, + "56": 2536167424.0, + "57": 2536167424.0, + "58": 2536167424.0, + "59": 2536167424.0, + "60": 2536167424.0, + "61": 2536167424.0, + "62": 2536167424.0, + "63": 2536167424.0, + "64": 2536167424.0, + "65": 2536167424.0, + "66": 2536167424.0, + "67": 2536167424.0, + "68": 2536167424.0, + "69": 2536167424.0, + "70": 2536167424.0, + "71": 2536167424.0, + "72": 2536167424.0, + "73": 2536167424.0, + "74": 2536167424.0, + "75": 2536167424.0, + "76": 2536167424.0, + "77": 2536167424.0, + "78": 2536167424.0, + "79": 2536167424.0, + "80": 2536167424.0, + "81": 2536167424.0, + "82": 2536167424.0, + "83": 2536167424.0, + "84": 2536167424.0, + "85": 2536167424.0, + "86": 2536167424.0, + "87": 2536167424.0, + "88": 2536167424.0, + "89": 2536167424.0, + "90": 2536167424.0, + "91": 2536167424.0, + "92": 2536167424.0, + "93": 2536167424.0, + "94": 2536167424.0, + "95": 2536167424.0, + "96": 2536167424.0, + "97": 2536167424.0, + "98": 2536167424.0, + "99": 2536167424.0, + "100": 2536167424.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.55988, + "2": 0.14641, + "3": 0.12458, + "4": 0.12359, + "5": 0.12323, + "6": 0.12296, + "7": 0.12273, + "8": 0.12322, + "9": 0.12253, + "10": 0.12257, + "11": 0.12186, + "12": 0.12242, + "13": 0.12302, + "14": 0.12233, + "15": 0.12201, + "16": 0.12138, + "17": 0.12155, + "18": 0.12113, + "19": 0.12159, + "20": 0.12132, + "21": 0.12202, + "22": 0.12206, + "23": 0.12186, + "24": 0.12276, + "25": 0.12238, + "26": 0.122, + "27": 0.12177, + "28": 0.12203, + "29": 0.12255, + "30": 0.12285, + "31": 0.1224, + "32": 0.12276, + "33": 0.12359, + "34": 0.12194, + "35": 0.12249, + "36": 0.12276, + "37": 0.12249, + "38": 0.12249, + "39": 0.12333, + "40": 0.12327, + "41": 0.12316, + "42": 0.12307, + "43": 0.12249, + "44": 0.12267, + "45": 0.12282, + "46": 0.12405, + "47": 0.12264, + "48": 0.12412, + "49": 0.12277, + "50": 0.12365, + "51": 0.1271, + "52": 0.12708, + "53": 0.12522, + "54": 0.1263, + "55": 0.12587, + "56": 0.12762, + "57": 0.12527, + "58": 0.12651, + "59": 0.12671, + "60": 0.12654, + "61": 0.12604, + "62": 0.12577, + "63": 0.12494, + "64": 0.12609, + "65": 0.12576, + "66": 0.12652, + "67": 0.12628, + "68": 0.12655, + "69": 0.12565, + "70": 0.12576, + "71": 0.12521, + "72": 0.12593, + "73": 0.12578, + "74": 0.12645, + "75": 0.12537, + "76": 0.12616, + "77": 0.12525, + "78": 0.12803, + "79": 0.1252, + "80": 0.12678, + "81": 0.12525, + "82": 0.12597, + "83": 0.12596, + "84": 0.12603, + "85": 0.1257, + "86": 0.12623, + "87": 0.12511, + "88": 0.12609, + "89": 0.12568, + "90": 0.12585, + "91": 0.12495, + "92": 0.12654, + "93": 0.12549, + "94": 0.12609, + "95": 0.12518, + "96": 0.12593, + "97": 0.12598, + "98": 0.12611, + "99": 0.12441, + "100": 0.12715 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json index f24760a1ccb..bd7ca46935f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.8401, "5": 10.84032, "10": 10.81341, "15": 10.80278, "20": 10.70496, "25": 10.53846, "30": 10.35517, "35": 10.27147, "40": 10.08044, "45": 9.82292, "50": 9.90113, "55": 9.86422, "60": 9.48029, "65": 8.93749, "70": 9.71025, "75": 9.40879, "80": 9.39077, "85": 9.59743, "90": 9.80386, "95": 9.50565, "100": 9.38812}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1670.0, "5": 1970.0, "10": 1436.0, "15": 1918.0, "20": 1786.0, "25": 1610.0, "30": 2039.0, "35": 2001.0, "40": 2321.0, "45": 2205.0, "50": 2365.0, "55": 2489.0, "60": 2508.0, "65": 2719.0, "70": 3241.0, "75": 2643.0, "80": 3368.0, "85": 3336.0, "90": 2961.0, "95": 3533.0, "100": 3432.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1230390272.0, "5": 1230390272.0, "10": 1230390272.0, "15": 1230390272.0, "20": 1230390272.0, "25": 1230390272.0, "30": 1230390272.0, "35": 1230390272.0, "40": 1230390272.0, "45": 1230390272.0, "50": 1230390272.0, "55": 1230390272.0, "60": 1230390272.0, "65": 1230390272.0, "70": 1230390272.0, "75": 1230390272.0, "80": 1230390272.0, "85": 1230390272.0, "90": 1230390272.0, "95": 1230390272.0, "100": 1230390272.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1984492032.0, "5": 2531972608.0, "10": 2531972608.0, "15": 2531972608.0, "20": 2531972608.0, "25": 2531972608.0, "30": 2531972608.0, "35": 2531972608.0, "40": 2531972608.0, "45": 2531972608.0, "50": 2531972608.0, "55": 2531972608.0, "60": 2531972608.0, "65": 2531972608.0, "70": 2531972608.0, "75": 2531972608.0, "80": 2531972608.0, "85": 2531972608.0, "90": 2531972608.0, "95": 2531972608.0, "100": 2531972608.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 6.64146, "5": 0.13017, "10": 0.12892, "15": 0.12866, "20": 0.12752, "25": 0.12949, "30": 0.12766, "35": 0.12823, "40": 0.12721, "45": 0.12802, "50": 0.12653, "55": 0.13387, "60": 0.12431, "65": 0.12501, "70": 0.12535, "75": 0.12473, "80": 0.12431, "85": 0.12451, "90": 0.12436, "95": 0.12406, "100": 0.12423}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84449, + "13": 10.86322, + "14": 10.86353, + "15": 10.79981, + "16": 10.79262, + "17": 10.77477, + "18": 10.80157, + "19": 10.79148, + "20": 10.70508, + "21": 10.68176, + "22": 10.56548, + "23": 10.70147, + "24": 10.57889, + "25": 10.53597, + "26": 10.60742, + "27": 10.59423, + "28": 10.56119, + "29": 10.57569, + "30": 10.35474, + "31": 10.12616, + "32": 10.46566, + "33": 10.45233, + "34": 10.22493, + "35": 10.27091, + "36": 10.22168, + "37": 10.33936, + "38": 10.18641, + "39": 10.39431, + "40": 10.07792, + "41": 10.13872, + "42": 10.20182, + "43": 9.83818, + "44": 9.94274, + "45": 9.82303, + "46": 9.82185, + "47": 10.13443, + "48": 9.84098, + "49": 9.52095, + "50": 9.90109, + "51": 9.83457, + "52": 9.73232, + "53": 10.0488, + "54": 9.93895, + "55": 9.863, + "56": 9.613, + "57": 9.46966, + "58": 9.81135, + "59": 9.57107, + "60": 9.48155, + "61": 9.6788, + "62": 9.96581, + "63": 9.35273, + "64": 9.75648, + "65": 8.93771, + "66": 9.68153, + "67": 9.35671, + "68": 9.76807, + "69": 9.7739, + "70": 9.71016, + "71": 9.60009, + "72": 9.56793, + "73": 9.4774, + "74": 8.93177, + "75": 9.4072, + "76": 9.06849, + "77": 10.0464, + "78": 9.70988, + "79": 9.35733, + "80": 9.38975, + "81": 9.4662, + "82": 9.68058, + "83": 9.2914, + "84": 9.40191, + "85": 9.59735, + "86": 9.06209, + "87": 9.57922, + "88": 9.73259, + "89": 9.58836, + "90": 9.80354, + "91": 9.31991, + "92": 9.35011, + "93": 9.06382, + "94": 8.81909, + "95": 9.50568, + "96": 9.51071, + "97": 9.29241, + "98": 9.65578, + "99": 8.87401, + "100": 9.38833 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1966.0, + "12": 1767.0, + "13": 1876.0, + "14": 1881.0, + "15": 1928.0, + "16": 1875.0, + "17": 1790.0, + "18": 1736.0, + "19": 1804.0, + "20": 1690.0, + "21": 2008.0, + "22": 1765.0, + "23": 2073.0, + "24": 1618.0, + "25": 1720.0, + "26": 1807.0, + "27": 1861.0, + "28": 2026.0, + "29": 1982.0, + "30": 1981.0, + "31": 1688.0, + "32": 1913.0, + "33": 2123.0, + "34": 1893.0, + "35": 2007.0, + "36": 1987.0, + "37": 2334.0, + "38": 2223.0, + "39": 2417.0, + "40": 2370.0, + "41": 2352.0, + "42": 2269.0, + "43": 1967.0, + "44": 2183.0, + "45": 2150.0, + "46": 2350.0, + "47": 2555.0, + "48": 2463.0, + "49": 2326.0, + "50": 2270.0, + "51": 2508.0, + "52": 2495.0, + "53": 2856.0, + "54": 2692.0, + "55": 2482.0, + "56": 2614.0, + "57": 2283.0, + "58": 2894.0, + "59": 2659.0, + "60": 2561.0, + "61": 3006.0, + "62": 2671.0, + "63": 2488.0, + "64": 3092.0, + "65": 2622.0, + "66": 3108.0, + "67": 2741.0, + "68": 2942.0, + "69": 2983.0, + "70": 3347.0, + "71": 3034.0, + "72": 2438.0, + "73": 3075.0, + "74": 1931.0, + "75": 2722.0, + "76": 2960.0, + "77": 3387.0, + "78": 3268.0, + "79": 3079.0, + "80": 3404.0, + "81": 3674.0, + "82": 3192.0, + "83": 2791.0, + "84": 3224.0, + "85": 3237.0, + "86": 2646.0, + "87": 3840.0, + "88": 3114.0, + "89": 3410.0, + "90": 3184.0, + "91": 3073.0, + "92": 3396.0, + "93": 2711.0, + "94": 3530.0, + "95": 3387.0, + "96": 3530.0, + "97": 3277.0, + "98": 3775.0, + "99": 3421.0, + "100": 3350.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1234585088.0, + "2": 1234585088.0, + "3": 1234585088.0, + "4": 1234585088.0, + "5": 1234585088.0, + "6": 1234585088.0, + "7": 1234585088.0, + "8": 1234585088.0, + "9": 1234585088.0, + "10": 1234585088.0, + "11": 1234585088.0, + "12": 1234585088.0, + "13": 1234585088.0, + "14": 1234585088.0, + "15": 1234585088.0, + "16": 1234585088.0, + "17": 1234585088.0, + "18": 1234585088.0, + "19": 1234585088.0, + "20": 1234585088.0, + "21": 1234585088.0, + "22": 1234585088.0, + "23": 1234585088.0, + "24": 1234585088.0, + "25": 1234585088.0, + "26": 1234585088.0, + "27": 1234585088.0, + "28": 1234585088.0, + "29": 1234585088.0, + "30": 1234585088.0, + "31": 1234585088.0, + "32": 1234585088.0, + "33": 1234585088.0, + "34": 1234585088.0, + "35": 1234585088.0, + "36": 1234585088.0, + "37": 1234585088.0, + "38": 1234585088.0, + "39": 1234585088.0, + "40": 1234585088.0, + "41": 1234585088.0, + "42": 1234585088.0, + "43": 1234585088.0, + "44": 1234585088.0, + "45": 1234585088.0, + "46": 1234585088.0, + "47": 1234585088.0, + "48": 1234585088.0, + "49": 1234585088.0, + "50": 1234585088.0, + "51": 1234585088.0, + "52": 1234585088.0, + "53": 1234585088.0, + "54": 1234585088.0, + "55": 1234585088.0, + "56": 1234585088.0, + "57": 1234585088.0, + "58": 1234585088.0, + "59": 1234585088.0, + "60": 1234585088.0, + "61": 1234585088.0, + "62": 1234585088.0, + "63": 1234585088.0, + "64": 1234585088.0, + "65": 1234585088.0, + "66": 1234585088.0, + "67": 1234585088.0, + "68": 1234585088.0, + "69": 1234585088.0, + "70": 1234585088.0, + "71": 1234585088.0, + "72": 1234585088.0, + "73": 1234585088.0, + "74": 1234585088.0, + "75": 1234585088.0, + "76": 1234585088.0, + "77": 1234585088.0, + "78": 1234585088.0, + "79": 1234585088.0, + "80": 1234585088.0, + "81": 1234585088.0, + "82": 1234585088.0, + "83": 1234585088.0, + "84": 1234585088.0, + "85": 1234585088.0, + "86": 1234585088.0, + "87": 1234585088.0, + "88": 1234585088.0, + "89": 1234585088.0, + "90": 1234585088.0, + "91": 1234585088.0, + "92": 1234585088.0, + "93": 1234585088.0, + "94": 1234585088.0, + "95": 1234585088.0, + "96": 1234585088.0, + "97": 1234585088.0, + "98": 1234585088.0, + "99": 1234585088.0, + "100": 1234585088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1984492544.0, + "2": 2536167424.0, + "3": 2536167424.0, + "4": 2536167424.0, + "5": 2536167424.0, + "6": 2536167424.0, + "7": 2536167424.0, + "8": 2536167424.0, + "9": 2536167424.0, + "10": 2536167424.0, + "11": 2536167424.0, + "12": 2536167424.0, + "13": 2536167424.0, + "14": 2536167424.0, + "15": 2536167424.0, + "16": 2536167424.0, + "17": 2536167424.0, + "18": 2536167424.0, + "19": 2536167424.0, + "20": 2536167424.0, + "21": 2536167424.0, + "22": 2536167424.0, + "23": 2536167424.0, + "24": 2536167424.0, + "25": 2536167424.0, + "26": 2536167424.0, + "27": 2536167424.0, + "28": 2536167424.0, + "29": 2536167424.0, + "30": 2536167424.0, + "31": 2536167424.0, + "32": 2536167424.0, + "33": 2536167424.0, + "34": 2536167424.0, + "35": 2536167424.0, + "36": 2536167424.0, + "37": 2536167424.0, + "38": 2536167424.0, + "39": 2536167424.0, + "40": 2536167424.0, + "41": 2536167424.0, + "42": 2536167424.0, + "43": 2536167424.0, + "44": 2536167424.0, + "45": 2536167424.0, + "46": 2536167424.0, + "47": 2536167424.0, + "48": 2536167424.0, + "49": 2536167424.0, + "50": 2536167424.0, + "51": 2536167424.0, + "52": 2536167424.0, + "53": 2536167424.0, + "54": 2536167424.0, + "55": 2536167424.0, + "56": 2536167424.0, + "57": 2536167424.0, + "58": 2536167424.0, + "59": 2536167424.0, + "60": 2536167424.0, + "61": 2536167424.0, + "62": 2536167424.0, + "63": 2536167424.0, + "64": 2536167424.0, + "65": 2536167424.0, + "66": 2536167424.0, + "67": 2536167424.0, + "68": 2536167424.0, + "69": 2536167424.0, + "70": 2536167424.0, + "71": 2536167424.0, + "72": 2536167424.0, + "73": 2536167424.0, + "74": 2536167424.0, + "75": 2536167424.0, + "76": 2536167424.0, + "77": 2536167424.0, + "78": 2536167424.0, + "79": 2536167424.0, + "80": 2536167424.0, + "81": 2536167424.0, + "82": 2536167424.0, + "83": 2536167424.0, + "84": 2536167424.0, + "85": 2536167424.0, + "86": 2536167424.0, + "87": 2536167424.0, + "88": 2536167424.0, + "89": 2536167424.0, + "90": 2536167424.0, + "91": 2536167424.0, + "92": 2536167424.0, + "93": 2536167424.0, + "94": 2536167424.0, + "95": 2536167424.0, + "96": 2536167424.0, + "97": 2536167424.0, + "98": 2536167424.0, + "99": 2536167424.0, + "100": 2536167424.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.43734, + "2": 0.14648, + "3": 0.12542, + "4": 0.12603, + "5": 0.12388, + "6": 0.12524, + "7": 0.12279, + "8": 0.1239, + "9": 0.12244, + "10": 0.12336, + "11": 0.12345, + "12": 0.12322, + "13": 0.12318, + "14": 0.12381, + "15": 0.12343, + "16": 0.12319, + "17": 0.12276, + "18": 0.12324, + "19": 0.12355, + "20": 0.12315, + "21": 0.12294, + "22": 0.12326, + "23": 0.12303, + "24": 0.12294, + "25": 0.12286, + "26": 0.12388, + "27": 0.12341, + "28": 0.12301, + "29": 0.12267, + "30": 0.12315, + "31": 0.12371, + "32": 0.12359, + "33": 0.12298, + "34": 0.12283, + "35": 0.12266, + "36": 0.12356, + "37": 0.12377, + "38": 0.12388, + "39": 0.12525, + "40": 0.12501, + "41": 0.12357, + "42": 0.12376, + "43": 0.12304, + "44": 0.12342, + "45": 0.12284, + "46": 0.12332, + "47": 0.12324, + "48": 0.12279, + "49": 0.12276, + "50": 0.12391, + "51": 0.12862, + "52": 0.12214, + "53": 0.12006, + "54": 0.12101, + "55": 0.12062, + "56": 0.12088, + "57": 0.121, + "58": 0.12034, + "59": 0.12049, + "60": 0.12066, + "61": 0.11974, + "62": 0.11979, + "63": 0.12196, + "64": 0.12149, + "65": 0.12119, + "66": 0.12067, + "67": 0.12079, + "68": 0.12104, + "69": 0.12025, + "70": 0.12059, + "71": 0.12069, + "72": 0.12102, + "73": 0.12115, + "74": 0.1208, + "75": 0.12051, + "76": 0.12011, + "77": 0.11958, + "78": 0.12095, + "79": 0.11983, + "80": 0.12106, + "81": 0.1203, + "82": 0.12062, + "83": 0.12021, + "84": 0.12036, + "85": 0.12053, + "86": 0.12119, + "87": 0.12057, + "88": 0.12092, + "89": 0.12271, + "90": 0.12095, + "91": 0.1204, + "92": 0.12052, + "93": 0.12075, + "94": 0.12025, + "95": 0.12129, + "96": 0.12087, + "97": 0.12098, + "98": 0.12136, + "99": 0.12046, + "100": 0.12064 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json new file mode 100644 index 00000000000..02ac3f23ca4 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84865, + "2": 10.84835, + "3": 10.84054, + "4": 10.82381, + "5": 10.86283, + "6": 10.87554, + "7": 10.87089, + "8": 10.85554, + "9": 10.86811, + "10": 10.82287, + "11": 10.90334, + "12": 10.87985, + "13": 10.8831, + "14": 10.89653, + "15": 10.81406, + "16": 10.81973, + "17": 10.79918, + "18": 10.82428, + "19": 10.8195, + "20": 10.71958, + "21": 10.67864, + "22": 10.53625, + "23": 10.72125, + "24": 10.57435, + "25": 10.52997, + "26": 10.60096, + "27": 10.61502, + "28": 10.57231, + "29": 10.58614, + "30": 10.33749, + "31": 10.06516, + "32": 10.46436, + "33": 10.43612, + "34": 10.17278, + "35": 10.24173, + "36": 10.19042, + "37": 10.32282, + "38": 10.14881, + "39": 10.37709, + "40": 10.05124, + "41": 10.11355, + "42": 10.17253, + "43": 9.76298, + "44": 9.89293, + "45": 9.7664, + "46": 9.7601, + "47": 10.09424, + "48": 9.78753, + "49": 9.454, + "50": 9.8548, + "51": 9.79157, + "52": 9.68731, + "53": 10.02181, + "54": 9.90398, + "55": 9.82389, + "56": 9.57081, + "57": 9.40818, + "58": 9.77678, + "59": 9.52729, + "60": 9.44284, + "61": 9.64071, + "62": 9.94046, + "63": 9.31099, + "64": 9.72506, + "65": 8.8916, + "66": 9.6525, + "67": 9.31718, + "68": 9.73957, + "69": 9.74304, + "70": 9.67942, + "71": 9.56228, + "72": 9.53149, + "73": 9.44531, + "74": 8.88431, + "75": 9.3677, + "76": 9.02482, + "77": 10.01647, + "78": 9.6813, + "79": 9.32719, + "80": 9.3577, + "81": 9.43335, + "82": 9.64804, + "83": 9.25573, + "84": 9.36738, + "85": 9.56091, + "86": 9.03567, + "87": 9.54622, + "88": 9.70041, + "89": 9.54992, + "90": 9.77126, + "91": 9.28801, + "92": 9.31055, + "93": 9.03195, + "94": 8.78121, + "95": 9.48115, + "96": 9.4759, + "97": 9.2489, + "98": 9.61705, + "99": 8.8368, + "100": 9.35043 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1658.0, + "2": 1892.0, + "3": 1844.0, + "4": 1851.0, + "5": 1895.0, + "6": 2058.0, + "7": 1902.0, + "8": 1808.0, + "9": 1894.0, + "10": 1583.0, + "11": 2044.0, + "12": 1889.0, + "13": 2101.0, + "14": 2062.0, + "15": 1981.0, + "16": 1922.0, + "17": 1845.0, + "18": 1842.0, + "19": 1890.0, + "20": 1769.0, + "21": 2048.0, + "22": 1800.0, + "23": 2134.0, + "24": 1822.0, + "25": 1861.0, + "26": 1864.0, + "27": 1886.0, + "28": 2050.0, + "29": 2009.0, + "30": 2118.0, + "31": 1688.0, + "32": 2112.0, + "33": 2153.0, + "34": 1963.0, + "35": 2014.0, + "36": 2023.0, + "37": 2222.0, + "38": 2239.0, + "39": 2367.0, + "40": 2161.0, + "41": 2422.0, + "42": 2240.0, + "43": 2116.0, + "44": 2343.0, + "45": 2127.0, + "46": 2189.0, + "47": 2411.0, + "48": 2347.0, + "49": 2271.0, + "50": 2345.0, + "51": 2482.0, + "52": 2570.0, + "53": 2835.0, + "54": 2589.0, + "55": 2450.0, + "56": 2744.0, + "57": 2429.0, + "58": 2684.0, + "59": 2748.0, + "60": 2464.0, + "61": 2995.0, + "62": 2518.0, + "63": 2570.0, + "64": 2843.0, + "65": 2648.0, + "66": 2842.0, + "67": 2954.0, + "68": 2833.0, + "69": 3027.0, + "70": 2993.0, + "71": 3010.0, + "72": 2597.0, + "73": 3002.0, + "74": 2325.0, + "75": 2882.0, + "76": 3143.0, + "77": 3062.0, + "78": 3272.0, + "79": 3303.0, + "80": 3280.0, + "81": 3517.0, + "82": 3283.0, + "83": 2834.0, + "84": 3365.0, + "85": 3288.0, + "86": 2562.0, + "87": 3493.0, + "88": 3388.0, + "89": 3102.0, + "90": 3230.0, + "91": 3154.0, + "92": 3263.0, + "93": 2967.0, + "94": 3520.0, + "95": 3175.0, + "96": 3317.0, + "97": 2999.0, + "98": 3549.0, + "99": 3248.0, + "100": 3227.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 892293120.0, + "2": 892293120.0, + "3": 892293120.0, + "4": 892293120.0, + "5": 892293120.0, + "6": 892293120.0, + "7": 892293120.0, + "8": 892293120.0, + "9": 892293120.0, + "10": 892293120.0, + "11": 892293120.0, + "12": 892293120.0, + "13": 892293120.0, + "14": 892293120.0, + "15": 892293120.0, + "16": 892293120.0, + "17": 892293120.0, + "18": 892293120.0, + "19": 892293120.0, + "20": 892293120.0, + "21": 892293120.0, + "22": 892293120.0, + "23": 892293120.0, + "24": 892293120.0, + "25": 892293120.0, + "26": 892293120.0, + "27": 892293120.0, + "28": 892293120.0, + "29": 892293120.0, + "30": 892293120.0, + "31": 892293120.0, + "32": 892293120.0, + "33": 892293120.0, + "34": 892293120.0, + "35": 892293120.0, + "36": 892293120.0, + "37": 892293120.0, + "38": 892293120.0, + "39": 892293120.0, + "40": 892293120.0, + "41": 892293120.0, + "42": 892293120.0, + "43": 892293120.0, + "44": 892293120.0, + "45": 892293120.0, + "46": 892293120.0, + "47": 892293120.0, + "48": 892293120.0, + "49": 892293120.0, + "50": 892293120.0, + "51": 892293120.0, + "52": 892293120.0, + "53": 892293120.0, + "54": 892293120.0, + "55": 892293120.0, + "56": 892293120.0, + "57": 892293120.0, + "58": 892293120.0, + "59": 892293120.0, + "60": 892293120.0, + "61": 892293120.0, + "62": 892293120.0, + "63": 892293120.0, + "64": 892293120.0, + "65": 892293120.0, + "66": 892293120.0, + "67": 892293120.0, + "68": 892293120.0, + "69": 892293120.0, + "70": 892293120.0, + "71": 892293120.0, + "72": 892293120.0, + "73": 892293120.0, + "74": 892293120.0, + "75": 892293120.0, + "76": 892293120.0, + "77": 892293120.0, + "78": 892293120.0, + "79": 892293120.0, + "80": 892293120.0, + "81": 892293120.0, + "82": 892293120.0, + "83": 892293120.0, + "84": 892293120.0, + "85": 892293120.0, + "86": 892293120.0, + "87": 892293120.0, + "88": 892293120.0, + "89": 892293120.0, + "90": 892293120.0, + "91": 892293120.0, + "92": 892293120.0, + "93": 892293120.0, + "94": 892293120.0, + "95": 892293120.0, + "96": 892293120.0, + "97": 892293120.0, + "98": 892293120.0, + "99": 892293120.0, + "100": 892293120.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2236674048.0, + "2": 2599285760.0, + "3": 2599285760.0, + "4": 2599285760.0, + "5": 2599285760.0, + "6": 2599285760.0, + "7": 2599285760.0, + "8": 2599285760.0, + "9": 2599285760.0, + "10": 2599285760.0, + "11": 2599285760.0, + "12": 2599285760.0, + "13": 2599285760.0, + "14": 2599285760.0, + "15": 2599285760.0, + "16": 2599285760.0, + "17": 2599285760.0, + "18": 2599285760.0, + "19": 2599285760.0, + "20": 2599285760.0, + "21": 2599285760.0, + "22": 2599285760.0, + "23": 2599285760.0, + "24": 2599285760.0, + "25": 2599285760.0, + "26": 2599285760.0, + "27": 2599285760.0, + "28": 2599285760.0, + "29": 2599285760.0, + "30": 2599285760.0, + "31": 2599285760.0, + "32": 2599285760.0, + "33": 2599285760.0, + "34": 2599285760.0, + "35": 2599285760.0, + "36": 2599285760.0, + "37": 2599285760.0, + "38": 2599285760.0, + "39": 2599285760.0, + "40": 2599285760.0, + "41": 2599285760.0, + "42": 2599285760.0, + "43": 2599285760.0, + "44": 2599285760.0, + "45": 2599285760.0, + "46": 2599285760.0, + "47": 2599285760.0, + "48": 2599285760.0, + "49": 2599285760.0, + "50": 2599285760.0, + "51": 2599285760.0, + "52": 2599285760.0, + "53": 2599285760.0, + "54": 2599285760.0, + "55": 2599285760.0, + "56": 2599285760.0, + "57": 2599285760.0, + "58": 2599285760.0, + "59": 2599285760.0, + "60": 2599285760.0, + "61": 2599285760.0, + "62": 2599285760.0, + "63": 2599285760.0, + "64": 2599285760.0, + "65": 2599285760.0, + "66": 2599285760.0, + "67": 2599285760.0, + "68": 2599285760.0, + "69": 2599285760.0, + "70": 2599285760.0, + "71": 2599285760.0, + "72": 2599285760.0, + "73": 2599285760.0, + "74": 2599285760.0, + "75": 2599285760.0, + "76": 2599285760.0, + "77": 2599285760.0, + "78": 2599285760.0, + "79": 2599285760.0, + "80": 2599285760.0, + "81": 2599285760.0, + "82": 2599285760.0, + "83": 2599285760.0, + "84": 2599285760.0, + "85": 2599285760.0, + "86": 2599285760.0, + "87": 2599285760.0, + "88": 2599285760.0, + "89": 2599285760.0, + "90": 2599285760.0, + "91": 2599285760.0, + "92": 2599285760.0, + "93": 2599285760.0, + "94": 2599285760.0, + "95": 2599285760.0, + "96": 2599285760.0, + "97": 2599285760.0, + "98": 2599285760.0, + "99": 2599285760.0, + "100": 2599285760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.33769, + "2": 0.18532, + "3": 0.14816, + "4": 0.14635, + "5": 0.14643, + "6": 0.14673, + "7": 0.1465, + "8": 0.14731, + "9": 0.14559, + "10": 0.1459, + "11": 0.14535, + "12": 0.14613, + "13": 0.14618, + "14": 0.14642, + "15": 0.1462, + "16": 0.14548, + "17": 0.14502, + "18": 0.1449, + "19": 0.145, + "20": 0.14696, + "21": 0.14496, + "22": 0.14536, + "23": 0.14489, + "24": 0.14464, + "25": 0.14477, + "26": 0.14552, + "27": 0.14534, + "28": 0.14506, + "29": 0.14487, + "30": 0.14509, + "31": 0.14642, + "32": 0.14499, + "33": 0.14538, + "34": 0.14461, + "35": 0.14541, + "36": 0.14464, + "37": 0.14539, + "38": 0.14569, + "39": 0.14515, + "40": 0.14583, + "41": 0.14836, + "42": 0.14589, + "43": 0.14625, + "44": 0.14559, + "45": 0.14588, + "46": 0.14644, + "47": 0.146, + "48": 0.1459, + "49": 0.14609, + "50": 0.14597, + "51": 0.15206, + "52": 0.1459, + "53": 0.1452, + "54": 0.14745, + "55": 0.14578, + "56": 0.14459, + "57": 0.14524, + "58": 0.14545, + "59": 0.14527, + "60": 0.14448, + "61": 0.14539, + "62": 0.14462, + "63": 0.14474, + "64": 0.1447, + "65": 0.14564, + "66": 0.14463, + "67": 0.14466, + "68": 0.14483, + "69": 0.14562, + "70": 0.1456, + "71": 0.14516, + "72": 0.14481, + "73": 0.14539, + "74": 0.14568, + "75": 0.14464, + "76": 0.14465, + "77": 0.14427, + "78": 0.14541, + "79": 0.1445, + "80": 0.14535, + "81": 0.14526, + "82": 0.14617, + "83": 0.14445, + "84": 0.14483, + "85": 0.14457, + "86": 0.14459, + "87": 0.14462, + "88": 0.14433, + "89": 0.14514, + "90": 0.14416, + "91": 0.14667, + "92": 0.14432, + "93": 0.14551, + "94": 0.14453, + "95": 0.14488, + "96": 0.14441, + "97": 0.14545, + "98": 0.14459, + "99": 0.14481, + "100": 0.14918 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json index d30c8d5b5e6..48aee8d379f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.84474, - "5": 10.86418, - "10": 10.82155, - "15": 10.81195, - "20": 10.71872, - "25": 10.53036, - "30": 10.3358, - "35": 10.24081, - "40": 10.05008, - "45": 9.76762, - "50": 9.85505, - "55": 9.82465, - "60": 9.44306, - "65": 8.89104, - "70": 9.67902, - "75": 9.36836, - "80": 9.35799, - "85": 9.56032, - "90": 9.77055, - "95": 9.48101, - "100": 9.34997 + "1": 10.84865, + "2": 10.84835, + "3": 10.84054, + "4": 10.82381, + "5": 10.86283, + "6": 10.87554, + "7": 10.87089, + "8": 10.85554, + "9": 10.86811, + "10": 10.82287, + "11": 10.90334, + "12": 10.87985, + "13": 10.8831, + "14": 10.89653, + "15": 10.81406, + "16": 10.81973, + "17": 10.79918, + "18": 10.82428, + "19": 10.8195, + "20": 10.71958, + "21": 10.67864, + "22": 10.53625, + "23": 10.72125, + "24": 10.57435, + "25": 10.52997, + "26": 10.60096, + "27": 10.61502, + "28": 10.57231, + "29": 10.58614, + "30": 10.33749, + "31": 10.06516, + "32": 10.46436, + "33": 10.43612, + "34": 10.17278, + "35": 10.24173, + "36": 10.19042, + "37": 10.32282, + "38": 10.14881, + "39": 10.37709, + "40": 10.05124, + "41": 10.11355, + "42": 10.17253, + "43": 9.76298, + "44": 9.89293, + "45": 9.7664, + "46": 9.7601, + "47": 10.09424, + "48": 9.78753, + "49": 9.454, + "50": 9.8548, + "51": 9.79157, + "52": 9.68731, + "53": 10.02181, + "54": 9.90398, + "55": 9.82389, + "56": 9.57081, + "57": 9.40818, + "58": 9.77678, + "59": 9.52729, + "60": 9.44284, + "61": 9.64071, + "62": 9.94046, + "63": 9.31099, + "64": 9.72506, + "65": 8.8916, + "66": 9.6525, + "67": 9.31718, + "68": 9.73957, + "69": 9.74304, + "70": 9.67942, + "71": 9.56228, + "72": 9.53149, + "73": 9.44531, + "74": 8.88431, + "75": 9.3677, + "76": 9.02482, + "77": 10.01647, + "78": 9.6813, + "79": 9.32719, + "80": 9.3577, + "81": 9.43335, + "82": 9.64804, + "83": 9.25573, + "84": 9.36738, + "85": 9.56091, + "86": 9.03567, + "87": 9.54622, + "88": 9.70041, + "89": 9.54992, + "90": 9.77126, + "91": 9.28801, + "92": 9.31055, + "93": 9.03195, + "94": 8.78121, + "95": 9.48115, + "96": 9.4759, + "97": 9.2489, + "98": 9.61705, + "99": 8.8368, + "100": 9.35043 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1776.0, - "5": 2128.0, - "10": 1615.0, - "15": 2021.0, - "20": 1775.0, - "25": 1916.0, - "30": 2029.0, - "35": 2107.0, - "40": 2174.0, - "45": 2110.0, - "50": 2363.0, - "55": 2460.0, - "60": 2462.0, - "65": 2724.0, - "70": 2952.0, - "75": 2823.0, - "80": 3222.0, - "85": 3314.0, - "90": 3087.0, - "95": 3146.0, - "100": 3331.0 + "1": 1658.0, + "2": 1892.0, + "3": 1844.0, + "4": 1851.0, + "5": 1895.0, + "6": 2058.0, + "7": 1902.0, + "8": 1808.0, + "9": 1894.0, + "10": 1583.0, + "11": 2044.0, + "12": 1889.0, + "13": 2101.0, + "14": 2062.0, + "15": 1981.0, + "16": 1922.0, + "17": 1845.0, + "18": 1842.0, + "19": 1890.0, + "20": 1769.0, + "21": 2048.0, + "22": 1800.0, + "23": 2134.0, + "24": 1822.0, + "25": 1861.0, + "26": 1864.0, + "27": 1886.0, + "28": 2050.0, + "29": 2009.0, + "30": 2118.0, + "31": 1688.0, + "32": 2112.0, + "33": 2153.0, + "34": 1963.0, + "35": 2014.0, + "36": 2023.0, + "37": 2222.0, + "38": 2239.0, + "39": 2367.0, + "40": 2161.0, + "41": 2422.0, + "42": 2240.0, + "43": 2116.0, + "44": 2343.0, + "45": 2127.0, + "46": 2189.0, + "47": 2411.0, + "48": 2347.0, + "49": 2271.0, + "50": 2345.0, + "51": 2482.0, + "52": 2570.0, + "53": 2835.0, + "54": 2589.0, + "55": 2450.0, + "56": 2744.0, + "57": 2429.0, + "58": 2684.0, + "59": 2748.0, + "60": 2464.0, + "61": 2995.0, + "62": 2518.0, + "63": 2570.0, + "64": 2843.0, + "65": 2648.0, + "66": 2842.0, + "67": 2954.0, + "68": 2833.0, + "69": 3027.0, + "70": 2993.0, + "71": 3010.0, + "72": 2597.0, + "73": 3002.0, + "74": 2325.0, + "75": 2882.0, + "76": 3143.0, + "77": 3062.0, + "78": 3272.0, + "79": 3303.0, + "80": 3280.0, + "81": 3517.0, + "82": 3283.0, + "83": 2834.0, + "84": 3365.0, + "85": 3288.0, + "86": 2562.0, + "87": 3493.0, + "88": 3388.0, + "89": 3102.0, + "90": 3230.0, + "91": 3154.0, + "92": 3263.0, + "93": 2967.0, + "94": 3520.0, + "95": 3175.0, + "96": 3317.0, + "97": 2999.0, + "98": 3549.0, + "99": 3248.0, + "100": 3227.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 888098304.0, - "5": 888098304.0, - "10": 888098304.0, - "15": 888098304.0, - "20": 888098304.0, - "25": 888098304.0, - "30": 888098304.0, - "35": 888098304.0, - "40": 888098304.0, - "45": 888098304.0, - "50": 888098304.0, - "55": 888098304.0, - "60": 888098304.0, - "65": 888098304.0, - "70": 888098304.0, - "75": 888098304.0, - "80": 888098304.0, - "85": 888098304.0, - "90": 888098304.0, - "95": 888098304.0, - "100": 888098304.0 + "1": 892293120.0, + "2": 892293120.0, + "3": 892293120.0, + "4": 892293120.0, + "5": 892293120.0, + "6": 892293120.0, + "7": 892293120.0, + "8": 892293120.0, + "9": 892293120.0, + "10": 892293120.0, + "11": 892293120.0, + "12": 892293120.0, + "13": 892293120.0, + "14": 892293120.0, + "15": 892293120.0, + "16": 892293120.0, + "17": 892293120.0, + "18": 892293120.0, + "19": 892293120.0, + "20": 892293120.0, + "21": 892293120.0, + "22": 892293120.0, + "23": 892293120.0, + "24": 892293120.0, + "25": 892293120.0, + "26": 892293120.0, + "27": 892293120.0, + "28": 892293120.0, + "29": 892293120.0, + "30": 892293120.0, + "31": 892293120.0, + "32": 892293120.0, + "33": 892293120.0, + "34": 892293120.0, + "35": 892293120.0, + "36": 892293120.0, + "37": 892293120.0, + "38": 892293120.0, + "39": 892293120.0, + "40": 892293120.0, + "41": 892293120.0, + "42": 892293120.0, + "43": 892293120.0, + "44": 892293120.0, + "45": 892293120.0, + "46": 892293120.0, + "47": 892293120.0, + "48": 892293120.0, + "49": 892293120.0, + "50": 892293120.0, + "51": 892293120.0, + "52": 892293120.0, + "53": 892293120.0, + "54": 892293120.0, + "55": 892293120.0, + "56": 892293120.0, + "57": 892293120.0, + "58": 892293120.0, + "59": 892293120.0, + "60": 892293120.0, + "61": 892293120.0, + "62": 892293120.0, + "63": 892293120.0, + "64": 892293120.0, + "65": 892293120.0, + "66": 892293120.0, + "67": 892293120.0, + "68": 892293120.0, + "69": 892293120.0, + "70": 892293120.0, + "71": 892293120.0, + "72": 892293120.0, + "73": 892293120.0, + "74": 892293120.0, + "75": 892293120.0, + "76": 892293120.0, + "77": 892293120.0, + "78": 892293120.0, + "79": 892293120.0, + "80": 892293120.0, + "81": 892293120.0, + "82": 892293120.0, + "83": 892293120.0, + "84": 892293120.0, + "85": 892293120.0, + "86": 892293120.0, + "87": 892293120.0, + "88": 892293120.0, + "89": 892293120.0, + "90": 892293120.0, + "91": 892293120.0, + "92": 892293120.0, + "93": 892293120.0, + "94": 892293120.0, + "95": 892293120.0, + "96": 892293120.0, + "97": 892293120.0, + "98": 892293120.0, + "99": 892293120.0, + "100": 892293120.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 3216302592.0, - "5": 3575768576.0, - "10": 3575768576.0, - "15": 3575768576.0, - "20": 3575768576.0, - "25": 3575768576.0, - "30": 3575768576.0, - "35": 3575768576.0, - "40": 3575768576.0, - "45": 3575768576.0, - "50": 3575768576.0, - "55": 3575768576.0, - "60": 3575768576.0, - "65": 3575768576.0, - "70": 3575768576.0, - "75": 3575768576.0, - "80": 3575768576.0, - "85": 3575768576.0, - "90": 3575768576.0, - "95": 3575768576.0, - "100": 3575768576.0 + "1": 2236674048.0, + "2": 2599285760.0, + "3": 2599285760.0, + "4": 2599285760.0, + "5": 2599285760.0, + "6": 2599285760.0, + "7": 2599285760.0, + "8": 2599285760.0, + "9": 2599285760.0, + "10": 2599285760.0, + "11": 2599285760.0, + "12": 2599285760.0, + "13": 2599285760.0, + "14": 2599285760.0, + "15": 2599285760.0, + "16": 2599285760.0, + "17": 2599285760.0, + "18": 2599285760.0, + "19": 2599285760.0, + "20": 2599285760.0, + "21": 2599285760.0, + "22": 2599285760.0, + "23": 2599285760.0, + "24": 2599285760.0, + "25": 2599285760.0, + "26": 2599285760.0, + "27": 2599285760.0, + "28": 2599285760.0, + "29": 2599285760.0, + "30": 2599285760.0, + "31": 2599285760.0, + "32": 2599285760.0, + "33": 2599285760.0, + "34": 2599285760.0, + "35": 2599285760.0, + "36": 2599285760.0, + "37": 2599285760.0, + "38": 2599285760.0, + "39": 2599285760.0, + "40": 2599285760.0, + "41": 2599285760.0, + "42": 2599285760.0, + "43": 2599285760.0, + "44": 2599285760.0, + "45": 2599285760.0, + "46": 2599285760.0, + "47": 2599285760.0, + "48": 2599285760.0, + "49": 2599285760.0, + "50": 2599285760.0, + "51": 2599285760.0, + "52": 2599285760.0, + "53": 2599285760.0, + "54": 2599285760.0, + "55": 2599285760.0, + "56": 2599285760.0, + "57": 2599285760.0, + "58": 2599285760.0, + "59": 2599285760.0, + "60": 2599285760.0, + "61": 2599285760.0, + "62": 2599285760.0, + "63": 2599285760.0, + "64": 2599285760.0, + "65": 2599285760.0, + "66": 2599285760.0, + "67": 2599285760.0, + "68": 2599285760.0, + "69": 2599285760.0, + "70": 2599285760.0, + "71": 2599285760.0, + "72": 2599285760.0, + "73": 2599285760.0, + "74": 2599285760.0, + "75": 2599285760.0, + "76": 2599285760.0, + "77": 2599285760.0, + "78": 2599285760.0, + "79": 2599285760.0, + "80": 2599285760.0, + "81": 2599285760.0, + "82": 2599285760.0, + "83": 2599285760.0, + "84": 2599285760.0, + "85": 2599285760.0, + "86": 2599285760.0, + "87": 2599285760.0, + "88": 2599285760.0, + "89": 2599285760.0, + "90": 2599285760.0, + "91": 2599285760.0, + "92": 2599285760.0, + "93": 2599285760.0, + "94": 2599285760.0, + "95": 2599285760.0, + "96": 2599285760.0, + "97": 2599285760.0, + "98": 2599285760.0, + "99": 2599285760.0, + "100": 2599285760.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.26216, - "5": 0.17522, - "10": 0.17308, - "15": 0.17129, - "20": 0.17153, - "25": 0.17072, - "30": 0.17154, - "35": 0.17096, - "40": 0.16956, - "45": 0.17006, - "50": 0.16739, - "55": 0.17674, - "60": 0.17809, - "65": 0.17803, - "70": 0.17656, - "75": 0.17665, - "80": 0.17686, - "85": 0.17744, - "90": 0.17611, - "95": 0.17687, - "100": 0.1753 + "1": 9.51973, + "2": 0.20593, + "3": 0.14945, + "4": 0.14775, + "5": 0.14785, + "6": 0.14767, + "7": 0.14754, + "8": 0.14649, + "9": 0.14636, + "10": 0.14713, + "11": 0.14628, + "12": 0.14658, + "13": 0.14581, + "14": 0.14652, + "15": 0.14657, + "16": 0.14585, + "17": 0.14783, + "18": 0.1469, + "19": 0.14603, + "20": 0.14662, + "21": 0.14635, + "22": 0.1461, + "23": 0.14688, + "24": 0.14579, + "25": 0.14587, + "26": 0.14836, + "27": 0.14598, + "28": 0.1458, + "29": 0.14604, + "30": 0.14624, + "31": 0.14719, + "32": 0.14625, + "33": 0.14582, + "34": 0.14603, + "35": 0.14619, + "36": 0.14587, + "37": 0.14585, + "38": 0.14625, + "39": 0.14572, + "40": 0.14629, + "41": 0.14561, + "42": 0.14587, + "43": 0.14672, + "44": 0.14572, + "45": 0.14618, + "46": 0.14622, + "47": 0.14572, + "48": 0.14538, + "49": 0.14571, + "50": 0.1457, + "51": 0.1553, + "52": 0.14793, + "53": 0.14797, + "54": 0.14774, + "55": 0.14702, + "56": 0.15765, + "57": 0.1544, + "58": 0.15368, + "59": 0.15399, + "60": 0.15366, + "61": 0.15362, + "62": 0.15351, + "63": 0.15339, + "64": 0.15353, + "65": 0.15154, + "66": 0.14531, + "67": 0.14661, + "68": 0.14599, + "69": 0.14546, + "70": 0.14633, + "71": 0.14568, + "72": 0.1461, + "73": 0.14601, + "74": 0.1469, + "75": 0.14561, + "76": 0.14575, + "77": 0.14581, + "78": 0.14634, + "79": 0.14619, + "80": 0.14627, + "81": 0.146, + "82": 0.14559, + "83": 0.14618, + "84": 0.14683, + "85": 0.14582, + "86": 0.1462, + "87": 0.14574, + "88": 0.14574, + "89": 0.14516, + "90": 0.14556, + "91": 0.146, + "92": 0.14702, + "93": 0.14541, + "94": 0.14625, + "95": 0.14586, + "96": 0.1455, + "97": 0.14559, + "98": 0.14614, + "99": 0.15005, + "100": 0.14598 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json new file mode 100644 index 00000000000..92fea79bc3d --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79332, + "2": 10.80674, + "3": 10.8069, + "4": 10.77377, + "5": 10.84763, + "6": 10.86617, + "7": 10.8277, + "8": 10.81226, + "9": 10.83507, + "10": 10.77007, + "11": 10.89243, + "12": 10.84575, + "13": 10.85895, + "14": 10.883, + "15": 10.79054, + "16": 10.78048, + "17": 10.75666, + "18": 10.79491, + "19": 10.79656, + "20": 10.68058, + "21": 10.66001, + "22": 10.50257, + "23": 10.7118, + "24": 10.55327, + "25": 10.50697, + "26": 10.58286, + "27": 10.58643, + "28": 10.55758, + "29": 10.56227, + "30": 10.33257, + "31": 10.08654, + "32": 10.44724, + "33": 10.44499, + "34": 10.19999, + "35": 10.25465, + "36": 10.19443, + "37": 10.32044, + "38": 10.16641, + "39": 10.3774, + "40": 10.05603, + "41": 10.13739, + "42": 10.19161, + "43": 9.80954, + "44": 9.93054, + "45": 9.80619, + "46": 9.81395, + "47": 10.12881, + "48": 9.82729, + "49": 9.51291, + "50": 9.89126, + "51": 9.84055, + "52": 9.73438, + "53": 10.05482, + "54": 9.94058, + "55": 9.87124, + "56": 9.61045, + "57": 9.46116, + "58": 9.81654, + "59": 9.57887, + "60": 9.48507, + "61": 9.68515, + "62": 9.97438, + "63": 9.36298, + "64": 9.76793, + "65": 8.93913, + "66": 9.68918, + "67": 9.36638, + "68": 9.77507, + "69": 9.78344, + "70": 9.72196, + "71": 9.60806, + "72": 9.57714, + "73": 9.48934, + "74": 8.94008, + "75": 9.40867, + "76": 9.08075, + "77": 10.05717, + "78": 9.72281, + "79": 9.36465, + "80": 9.39746, + "81": 9.47553, + "82": 9.6886, + "83": 9.30263, + "84": 9.41008, + "85": 9.60793, + "86": 9.07115, + "87": 9.58676, + "88": 9.74129, + "89": 9.5986, + "90": 9.81041, + "91": 9.33113, + "92": 9.35502, + "93": 9.07481, + "94": 8.82745, + "95": 9.51149, + "96": 9.51876, + "97": 9.30173, + "98": 9.66726, + "99": 8.88087, + "100": 9.39727 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1624.0, + "2": 1758.0, + "3": 1678.0, + "4": 1680.0, + "5": 1824.0, + "6": 1899.0, + "7": 1871.0, + "8": 1677.0, + "9": 1802.0, + "10": 1414.0, + "11": 1904.0, + "12": 1630.0, + "13": 1956.0, + "14": 1777.0, + "15": 1902.0, + "16": 1801.0, + "17": 1830.0, + "18": 1630.0, + "19": 1835.0, + "20": 1642.0, + "21": 1752.0, + "22": 1696.0, + "23": 2085.0, + "24": 1618.0, + "25": 1593.0, + "26": 1722.0, + "27": 1778.0, + "28": 2042.0, + "29": 1900.0, + "30": 2001.0, + "31": 1592.0, + "32": 1757.0, + "33": 2116.0, + "34": 1924.0, + "35": 1904.0, + "36": 1852.0, + "37": 2382.0, + "38": 2195.0, + "39": 2267.0, + "40": 2335.0, + "41": 2223.0, + "42": 2317.0, + "43": 2069.0, + "44": 2060.0, + "45": 2140.0, + "46": 2397.0, + "47": 2464.0, + "48": 2455.0, + "49": 2276.0, + "50": 2374.0, + "51": 2574.0, + "52": 2457.0, + "53": 2905.0, + "54": 2609.0, + "55": 2220.0, + "56": 2663.0, + "57": 2258.0, + "58": 2898.0, + "59": 2676.0, + "60": 2397.0, + "61": 3048.0, + "62": 2533.0, + "63": 2370.0, + "64": 2975.0, + "65": 2591.0, + "66": 3065.0, + "67": 2732.0, + "68": 2870.0, + "69": 2955.0, + "70": 3112.0, + "71": 2989.0, + "72": 2451.0, + "73": 2881.0, + "74": 1859.0, + "75": 2649.0, + "76": 3026.0, + "77": 3316.0, + "78": 3212.0, + "79": 3183.0, + "80": 3262.0, + "81": 3669.0, + "82": 3187.0, + "83": 2798.0, + "84": 3209.0, + "85": 3309.0, + "86": 2738.0, + "87": 3804.0, + "88": 2989.0, + "89": 3327.0, + "90": 3031.0, + "91": 2720.0, + "92": 2972.0, + "93": 2719.0, + "94": 3387.0, + "95": 3321.0, + "96": 3342.0, + "97": 3191.0, + "98": 3533.0, + "99": 3214.0, + "100": 3318.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 716834304.0, + "2": 716834304.0, + "3": 716834304.0, + "4": 716834304.0, + "5": 716834304.0, + "6": 716834304.0, + "7": 716834304.0, + "8": 716834304.0, + "9": 716834304.0, + "10": 716834304.0, + "11": 716834304.0, + "12": 716834304.0, + "13": 716834304.0, + "14": 716834304.0, + "15": 716834304.0, + "16": 716834304.0, + "17": 716834304.0, + "18": 716834304.0, + "19": 716834304.0, + "20": 716834304.0, + "21": 716834304.0, + "22": 716834304.0, + "23": 716834304.0, + "24": 716834304.0, + "25": 716834304.0, + "26": 716834304.0, + "27": 716834304.0, + "28": 716834304.0, + "29": 716834304.0, + "30": 716834304.0, + "31": 716834304.0, + "32": 716834304.0, + "33": 716834304.0, + "34": 716834304.0, + "35": 716834304.0, + "36": 716834304.0, + "37": 716834304.0, + "38": 716834304.0, + "39": 716834304.0, + "40": 716834304.0, + "41": 716834304.0, + "42": 716834304.0, + "43": 716834304.0, + "44": 716834304.0, + "45": 716834304.0, + "46": 716834304.0, + "47": 716834304.0, + "48": 716834304.0, + "49": 716834304.0, + "50": 716834304.0, + "51": 716834304.0, + "52": 716834304.0, + "53": 716834304.0, + "54": 716834304.0, + "55": 716834304.0, + "56": 716834304.0, + "57": 716834304.0, + "58": 716834304.0, + "59": 716834304.0, + "60": 716834304.0, + "61": 716834304.0, + "62": 716834304.0, + "63": 716834304.0, + "64": 716834304.0, + "65": 716834304.0, + "66": 716834304.0, + "67": 716834304.0, + "68": 716834304.0, + "69": 716834304.0, + "70": 716834304.0, + "71": 716834304.0, + "72": 716834304.0, + "73": 716834304.0, + "74": 716834304.0, + "75": 716834304.0, + "76": 716834304.0, + "77": 716834304.0, + "78": 716834304.0, + "79": 716834304.0, + "80": 716834304.0, + "81": 716834304.0, + "82": 716834304.0, + "83": 716834304.0, + "84": 716834304.0, + "85": 716834304.0, + "86": 716834304.0, + "87": 716834304.0, + "88": 716834304.0, + "89": 716834304.0, + "90": 716834304.0, + "91": 716834304.0, + "92": 716834304.0, + "93": 716834304.0, + "94": 716834304.0, + "95": 716834304.0, + "96": 716834304.0, + "97": 716834304.0, + "98": 716834304.0, + "99": 716834304.0, + "100": 716834304.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910424576.0, + "2": 2193074176.0, + "3": 2193074176.0, + "4": 2193074176.0, + "5": 2193074176.0, + "6": 2193074176.0, + "7": 2193074176.0, + "8": 2193074176.0, + "9": 2193074176.0, + "10": 2193074176.0, + "11": 2193074176.0, + "12": 2193074176.0, + "13": 2193074176.0, + "14": 2193074176.0, + "15": 2193074176.0, + "16": 2193074176.0, + "17": 2193074176.0, + "18": 2193074176.0, + "19": 2193074176.0, + "20": 2193074176.0, + "21": 2193074176.0, + "22": 2193074176.0, + "23": 2193074176.0, + "24": 2193074176.0, + "25": 2193074176.0, + "26": 2193074176.0, + "27": 2193074176.0, + "28": 2193074176.0, + "29": 2193074176.0, + "30": 2193074176.0, + "31": 2193074176.0, + "32": 2193074176.0, + "33": 2193074176.0, + "34": 2193074176.0, + "35": 2193074176.0, + "36": 2193074176.0, + "37": 2193074176.0, + "38": 2193074176.0, + "39": 2193074176.0, + "40": 2193074176.0, + "41": 2193074176.0, + "42": 2193074176.0, + "43": 2193074176.0, + "44": 2193074176.0, + "45": 2193074176.0, + "46": 2193074176.0, + "47": 2193074176.0, + "48": 2193074176.0, + "49": 2193074176.0, + "50": 2193074176.0, + "51": 2193074176.0, + "52": 2193074176.0, + "53": 2193074176.0, + "54": 2193074176.0, + "55": 2193074176.0, + "56": 2193074176.0, + "57": 2193074176.0, + "58": 2193074176.0, + "59": 2193074176.0, + "60": 2193074176.0, + "61": 2193074176.0, + "62": 2193074176.0, + "63": 2193074176.0, + "64": 2193074176.0, + "65": 2193074176.0, + "66": 2193074176.0, + "67": 2193074176.0, + "68": 2193074176.0, + "69": 2193074176.0, + "70": 2193074176.0, + "71": 2193074176.0, + "72": 2193074176.0, + "73": 2193074176.0, + "74": 2193074176.0, + "75": 2193074176.0, + "76": 2193074176.0, + "77": 2193074176.0, + "78": 2193074176.0, + "79": 2193074176.0, + "80": 2193074176.0, + "81": 2193074176.0, + "82": 2193074176.0, + "83": 2193074176.0, + "84": 2193074176.0, + "85": 2193074176.0, + "86": 2193074176.0, + "87": 2193074176.0, + "88": 2193074176.0, + "89": 2193074176.0, + "90": 2193074176.0, + "91": 2193074176.0, + "92": 2193074176.0, + "93": 2193074176.0, + "94": 2193074176.0, + "95": 2193074176.0, + "96": 2193074176.0, + "97": 2193074176.0, + "98": 2193074176.0, + "99": 2193074176.0, + "100": 2193074176.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.17541, + "2": 0.17628, + "3": 0.14823, + "4": 0.14828, + "5": 0.14489, + "6": 0.14681, + "7": 0.14589, + "8": 0.14567, + "9": 0.14899, + "10": 0.14748, + "11": 0.1469, + "12": 0.14571, + "13": 0.14519, + "14": 0.14594, + "15": 0.14553, + "16": 0.1461, + "17": 0.14672, + "18": 0.14616, + "19": 0.1455, + "20": 0.14588, + "21": 0.14801, + "22": 0.14714, + "23": 0.14721, + "24": 0.14624, + "25": 0.1462, + "26": 0.14548, + "27": 0.14684, + "28": 0.14765, + "29": 0.14671, + "30": 0.14515, + "31": 0.14617, + "32": 0.14666, + "33": 0.14596, + "34": 0.14868, + "35": 0.14573, + "36": 0.14694, + "37": 0.14585, + "38": 0.14605, + "39": 0.14599, + "40": 0.14558, + "41": 0.14673, + "42": 0.14745, + "43": 0.1456, + "44": 0.14744, + "45": 0.14524, + "46": 0.14572, + "47": 0.14533, + "48": 0.14632, + "49": 0.14734, + "50": 0.1453, + "51": 0.16371, + "52": 0.14839, + "53": 0.17852, + "54": 0.14579, + "55": 0.14651, + "56": 0.14872, + "57": 0.14723, + "58": 0.14775, + "59": 0.14896, + "60": 0.14649, + "61": 0.14672, + "62": 0.14696, + "63": 0.14572, + "64": 0.14639, + "65": 0.14739, + "66": 0.14722, + "67": 0.14732, + "68": 0.14566, + "69": 0.14664, + "70": 0.14693, + "71": 0.14641, + "72": 0.14742, + "73": 0.14691, + "74": 0.1482, + "75": 0.15006, + "76": 0.146, + "77": 0.14585, + "78": 0.14677, + "79": 0.14716, + "80": 0.14605, + "81": 0.14678, + "82": 0.14648, + "83": 0.14624, + "84": 0.14639, + "85": 0.14622, + "86": 0.14829, + "87": 0.14591, + "88": 0.14541, + "89": 0.14865, + "90": 0.14587, + "91": 0.14618, + "92": 0.14625, + "93": 0.14624, + "94": 0.14583, + "95": 0.14675, + "96": 0.14876, + "97": 0.14645, + "98": 0.14588, + "99": 0.14617, + "100": 0.14618 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json index 4d218d51297..1246b8727ef 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.79205, "5": 10.84695, "10": 10.77106, "15": 10.79093, "20": 10.68042, "25": 10.50715, "30": 10.33325, "35": 10.25545, "40": 10.05544, "45": 9.80575, "50": 9.89082, "55": 9.87063, "60": 9.48478, "65": 8.94022, "70": 9.72243, "75": 9.40907, "80": 9.3976, "85": 9.60746, "90": 9.81041, "95": 9.5116, "100": 9.39722}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1580.0, "5": 1901.0, "10": 1346.0, "15": 1926.0, "20": 1643.0, "25": 1683.0, "30": 1867.0, "35": 2020.0, "40": 2252.0, "45": 2243.0, "50": 2459.0, "55": 2291.0, "60": 2404.0, "65": 2474.0, "70": 3102.0, "75": 2603.0, "80": 3420.0, "85": 3388.0, "90": 2904.0, "95": 3333.0, "100": 3347.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 714736640.0, "5": 714736640.0, "10": 714736640.0, "15": 714736640.0, "20": 714736640.0, "25": 714736640.0, "30": 714736640.0, "35": 714736640.0, "40": 714736640.0, "45": 714736640.0, "50": 714736640.0, "55": 714736640.0, "60": 714736640.0, "65": 714736640.0, "70": 714736640.0, "75": 714736640.0, "80": 714736640.0, "85": 714736640.0, "90": 714736640.0, "95": 714736640.0, "100": 714736640.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2399714304.0, "5": 2681315328.0, "10": 2681315328.0, "15": 2681315328.0, "20": 2681315328.0, "25": 2681315328.0, "30": 2681315328.0, "35": 2681315328.0, "40": 2681315328.0, "45": 2681315328.0, "50": 2681315328.0, "55": 2681315328.0, "60": 2681315328.0, "65": 2681315328.0, "70": 2681315328.0, "75": 2681315328.0, "80": 2681315328.0, "85": 2681315328.0, "90": 2681315328.0, "95": 2681315328.0, "100": 2681315328.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 9.04394, "5": 0.16449, "10": 0.16258, "15": 0.16166, "20": 0.18479, "25": 0.17915, "30": 0.18008, "35": 0.16013, "40": 0.15723, "45": 0.15769, "50": 0.15699, "55": 0.15776, "60": 0.15787, "65": 0.15913, "70": 0.15699, "75": 0.15748, "80": 0.15897, "85": 0.15762, "90": 0.15743, "95": 0.15733, "100": 0.15822}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79332, + "2": 10.80674, + "3": 10.8069, + "4": 10.77377, + "5": 10.84763, + "6": 10.86617, + "7": 10.8277, + "8": 10.81226, + "9": 10.83507, + "10": 10.77007, + "11": 10.89243, + "12": 10.84575, + "13": 10.85895, + "14": 10.883, + "15": 10.79054, + "16": 10.78048, + "17": 10.75666, + "18": 10.79491, + "19": 10.79656, + "20": 10.68058, + "21": 10.66001, + "22": 10.50257, + "23": 10.7118, + "24": 10.55327, + "25": 10.50697, + "26": 10.58286, + "27": 10.58643, + "28": 10.55758, + "29": 10.56227, + "30": 10.33257, + "31": 10.08654, + "32": 10.44724, + "33": 10.44499, + "34": 10.19999, + "35": 10.25465, + "36": 10.19443, + "37": 10.32044, + "38": 10.16641, + "39": 10.3774, + "40": 10.05603, + "41": 10.13739, + "42": 10.19161, + "43": 9.80954, + "44": 9.93054, + "45": 9.80619, + "46": 9.81395, + "47": 10.12881, + "48": 9.82729, + "49": 9.51291, + "50": 9.89126, + "51": 9.84055, + "52": 9.73438, + "53": 10.05482, + "54": 9.94058, + "55": 9.87124, + "56": 9.61045, + "57": 9.46116, + "58": 9.81654, + "59": 9.57887, + "60": 9.48507, + "61": 9.68515, + "62": 9.97438, + "63": 9.36298, + "64": 9.76793, + "65": 8.93913, + "66": 9.68918, + "67": 9.36638, + "68": 9.77507, + "69": 9.78344, + "70": 9.72196, + "71": 9.60806, + "72": 9.57714, + "73": 9.48934, + "74": 8.94008, + "75": 9.40867, + "76": 9.08075, + "77": 10.05717, + "78": 9.72281, + "79": 9.36465, + "80": 9.39746, + "81": 9.47553, + "82": 9.6886, + "83": 9.30263, + "84": 9.41008, + "85": 9.60793, + "86": 9.07115, + "87": 9.58676, + "88": 9.74129, + "89": 9.5986, + "90": 9.81041, + "91": 9.33113, + "92": 9.35502, + "93": 9.07481, + "94": 8.82745, + "95": 9.51149, + "96": 9.51876, + "97": 9.30173, + "98": 9.66726, + "99": 8.88087, + "100": 9.39727 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1624.0, + "2": 1758.0, + "3": 1678.0, + "4": 1680.0, + "5": 1824.0, + "6": 1899.0, + "7": 1871.0, + "8": 1677.0, + "9": 1802.0, + "10": 1414.0, + "11": 1904.0, + "12": 1630.0, + "13": 1956.0, + "14": 1777.0, + "15": 1902.0, + "16": 1801.0, + "17": 1830.0, + "18": 1630.0, + "19": 1835.0, + "20": 1642.0, + "21": 1752.0, + "22": 1696.0, + "23": 2085.0, + "24": 1618.0, + "25": 1593.0, + "26": 1722.0, + "27": 1778.0, + "28": 2042.0, + "29": 1900.0, + "30": 2001.0, + "31": 1592.0, + "32": 1757.0, + "33": 2116.0, + "34": 1924.0, + "35": 1904.0, + "36": 1852.0, + "37": 2382.0, + "38": 2195.0, + "39": 2267.0, + "40": 2335.0, + "41": 2223.0, + "42": 2317.0, + "43": 2069.0, + "44": 2060.0, + "45": 2140.0, + "46": 2397.0, + "47": 2464.0, + "48": 2455.0, + "49": 2276.0, + "50": 2374.0, + "51": 2574.0, + "52": 2457.0, + "53": 2905.0, + "54": 2609.0, + "55": 2220.0, + "56": 2663.0, + "57": 2258.0, + "58": 2898.0, + "59": 2676.0, + "60": 2397.0, + "61": 3048.0, + "62": 2533.0, + "63": 2370.0, + "64": 2975.0, + "65": 2591.0, + "66": 3065.0, + "67": 2732.0, + "68": 2870.0, + "69": 2955.0, + "70": 3112.0, + "71": 2989.0, + "72": 2451.0, + "73": 2881.0, + "74": 1859.0, + "75": 2649.0, + "76": 3026.0, + "77": 3316.0, + "78": 3212.0, + "79": 3183.0, + "80": 3262.0, + "81": 3669.0, + "82": 3187.0, + "83": 2798.0, + "84": 3209.0, + "85": 3309.0, + "86": 2738.0, + "87": 3804.0, + "88": 2989.0, + "89": 3327.0, + "90": 3031.0, + "91": 2720.0, + "92": 2972.0, + "93": 2719.0, + "94": 3387.0, + "95": 3321.0, + "96": 3342.0, + "97": 3191.0, + "98": 3533.0, + "99": 3214.0, + "100": 3318.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 716834304.0, + "2": 716834304.0, + "3": 716834304.0, + "4": 716834304.0, + "5": 716834304.0, + "6": 716834304.0, + "7": 716834304.0, + "8": 716834304.0, + "9": 716834304.0, + "10": 716834304.0, + "11": 716834304.0, + "12": 716834304.0, + "13": 716834304.0, + "14": 716834304.0, + "15": 716834304.0, + "16": 716834304.0, + "17": 716834304.0, + "18": 716834304.0, + "19": 716834304.0, + "20": 716834304.0, + "21": 716834304.0, + "22": 716834304.0, + "23": 716834304.0, + "24": 716834304.0, + "25": 716834304.0, + "26": 716834304.0, + "27": 716834304.0, + "28": 716834304.0, + "29": 716834304.0, + "30": 716834304.0, + "31": 716834304.0, + "32": 716834304.0, + "33": 716834304.0, + "34": 716834304.0, + "35": 716834304.0, + "36": 716834304.0, + "37": 716834304.0, + "38": 716834304.0, + "39": 716834304.0, + "40": 716834304.0, + "41": 716834304.0, + "42": 716834304.0, + "43": 716834304.0, + "44": 716834304.0, + "45": 716834304.0, + "46": 716834304.0, + "47": 716834304.0, + "48": 716834304.0, + "49": 716834304.0, + "50": 716834304.0, + "51": 716834304.0, + "52": 716834304.0, + "53": 716834304.0, + "54": 716834304.0, + "55": 716834304.0, + "56": 716834304.0, + "57": 716834304.0, + "58": 716834304.0, + "59": 716834304.0, + "60": 716834304.0, + "61": 716834304.0, + "62": 716834304.0, + "63": 716834304.0, + "64": 716834304.0, + "65": 716834304.0, + "66": 716834304.0, + "67": 716834304.0, + "68": 716834304.0, + "69": 716834304.0, + "70": 716834304.0, + "71": 716834304.0, + "72": 716834304.0, + "73": 716834304.0, + "74": 716834304.0, + "75": 716834304.0, + "76": 716834304.0, + "77": 716834304.0, + "78": 716834304.0, + "79": 716834304.0, + "80": 716834304.0, + "81": 716834304.0, + "82": 716834304.0, + "83": 716834304.0, + "84": 716834304.0, + "85": 716834304.0, + "86": 716834304.0, + "87": 716834304.0, + "88": 716834304.0, + "89": 716834304.0, + "90": 716834304.0, + "91": 716834304.0, + "92": 716834304.0, + "93": 716834304.0, + "94": 716834304.0, + "95": 716834304.0, + "96": 716834304.0, + "97": 716834304.0, + "98": 716834304.0, + "99": 716834304.0, + "100": 716834304.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910424576.0, + "2": 2193074176.0, + "3": 2193074176.0, + "4": 2193074176.0, + "5": 2193074176.0, + "6": 2193074176.0, + "7": 2193074176.0, + "8": 2193074176.0, + "9": 2193074176.0, + "10": 2193074176.0, + "11": 2193074176.0, + "12": 2193074176.0, + "13": 2193074176.0, + "14": 2193074176.0, + "15": 2193074176.0, + "16": 2193074176.0, + "17": 2193074176.0, + "18": 2193074176.0, + "19": 2193074176.0, + "20": 2193074176.0, + "21": 2193074176.0, + "22": 2193074176.0, + "23": 2193074176.0, + "24": 2193074176.0, + "25": 2193074176.0, + "26": 2193074176.0, + "27": 2193074176.0, + "28": 2193074176.0, + "29": 2193074176.0, + "30": 2193074176.0, + "31": 2193074176.0, + "32": 2193074176.0, + "33": 2193074176.0, + "34": 2193074176.0, + "35": 2193074176.0, + "36": 2193074176.0, + "37": 2193074176.0, + "38": 2193074176.0, + "39": 2193074176.0, + "40": 2193074176.0, + "41": 2193074176.0, + "42": 2193074176.0, + "43": 2193074176.0, + "44": 2193074176.0, + "45": 2193074176.0, + "46": 2193074176.0, + "47": 2193074176.0, + "48": 2193074176.0, + "49": 2193074176.0, + "50": 2193074176.0, + "51": 2193074176.0, + "52": 2193074176.0, + "53": 2193074176.0, + "54": 2193074176.0, + "55": 2193074176.0, + "56": 2193074176.0, + "57": 2193074176.0, + "58": 2193074176.0, + "59": 2193074176.0, + "60": 2193074176.0, + "61": 2193074176.0, + "62": 2193074176.0, + "63": 2193074176.0, + "64": 2193074176.0, + "65": 2193074176.0, + "66": 2193074176.0, + "67": 2193074176.0, + "68": 2193074176.0, + "69": 2193074176.0, + "70": 2193074176.0, + "71": 2193074176.0, + "72": 2193074176.0, + "73": 2193074176.0, + "74": 2193074176.0, + "75": 2193074176.0, + "76": 2193074176.0, + "77": 2193074176.0, + "78": 2193074176.0, + "79": 2193074176.0, + "80": 2193074176.0, + "81": 2193074176.0, + "82": 2193074176.0, + "83": 2193074176.0, + "84": 2193074176.0, + "85": 2193074176.0, + "86": 2193074176.0, + "87": 2193074176.0, + "88": 2193074176.0, + "89": 2193074176.0, + "90": 2193074176.0, + "91": 2193074176.0, + "92": 2193074176.0, + "93": 2193074176.0, + "94": 2193074176.0, + "95": 2193074176.0, + "96": 2193074176.0, + "97": 2193074176.0, + "98": 2193074176.0, + "99": 2193074176.0, + "100": 2193074176.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.43574, + "2": 0.18308, + "3": 0.16294, + "4": 0.15632, + "5": 0.15517, + "6": 0.15061, + "7": 0.15109, + "8": 0.1538, + "9": 0.15077, + "10": 0.15142, + "11": 0.15024, + "12": 0.15039, + "13": 0.14987, + "14": 0.15044, + "15": 0.1495, + "16": 0.15003, + "17": 0.14988, + "18": 0.1497, + "19": 0.15459, + "20": 0.15076, + "21": 0.1498, + "22": 0.15044, + "23": 0.15051, + "24": 0.15062, + "25": 0.14953, + "26": 0.15047, + "27": 0.14851, + "28": 0.14802, + "29": 0.14861, + "30": 0.1485, + "31": 0.1498, + "32": 0.14871, + "33": 0.1485, + "34": 0.14707, + "35": 0.14796, + "36": 0.14719, + "37": 0.15012, + "38": 0.14804, + "39": 0.1487, + "40": 0.14779, + "41": 0.14844, + "42": 0.1496, + "43": 0.15014, + "44": 0.14977, + "45": 0.1478, + "46": 0.14891, + "47": 0.14844, + "48": 0.1488, + "49": 0.14931, + "50": 0.14761, + "51": 0.15888, + "52": 0.1517, + "53": 0.14904, + "54": 0.17961, + "55": 0.14804, + "56": 0.1496, + "57": 0.1487, + "58": 0.14801, + "59": 0.14729, + "60": 0.14749, + "61": 0.14745, + "62": 0.1471, + "63": 0.14817, + "64": 0.1497, + "65": 0.14753, + "66": 0.14753, + "67": 0.14859, + "68": 0.14714, + "69": 0.14776, + "70": 0.14847, + "71": 0.14829, + "72": 0.14858, + "73": 0.14828, + "74": 0.14783, + "75": 0.14793, + "76": 0.14768, + "77": 0.14752, + "78": 0.14931, + "79": 0.15045, + "80": 0.14813, + "81": 0.1489, + "82": 0.1475, + "83": 0.14844, + "84": 0.1489, + "85": 0.14809, + "86": 0.14835, + "87": 0.14718, + "88": 0.14876, + "89": 0.14859, + "90": 0.1479, + "91": 0.14803, + "92": 0.14798, + "93": 0.14876, + "94": 0.14705, + "95": 0.14837, + "96": 0.14805, + "97": 0.14837, + "98": 0.14721, + "99": 0.14843, + "100": 0.14828 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json new file mode 100644 index 00000000000..801e633e981 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.90314, + "2": 10.88897, + "3": 10.89725, + "4": 10.88248, + "5": 10.89447, + "6": 10.91226, + "7": 10.89775, + "8": 10.88643, + "9": 10.89227, + "10": 10.8849, + "11": 10.91413, + "12": 10.88666, + "13": 10.89283, + "14": 10.90486, + "15": 10.83878, + "16": 10.84927, + "17": 10.83248, + "18": 10.83371, + "19": 10.83359, + "20": 10.74074, + "21": 10.7043, + "22": 10.59893, + "23": 10.72222, + "24": 10.60745, + "25": 10.57467, + "26": 10.62999, + "27": 10.62364, + "28": 10.57229, + "29": 10.6073, + "30": 10.37766, + "31": 10.15362, + "32": 10.47609, + "33": 10.48062, + "34": 10.24216, + "35": 10.29035, + "36": 10.25955, + "37": 10.36145, + "38": 10.21396, + "39": 10.44502, + "40": 10.11492, + "41": 10.1605, + "42": 10.23468, + "43": 9.85032, + "44": 9.97764, + "45": 9.85681, + "46": 9.8307, + "47": 10.17976, + "48": 9.85811, + "49": 9.54378, + "50": 9.93469, + "51": 9.86793, + "52": 9.76274, + "53": 10.10895, + "54": 9.95538, + "55": 9.8756, + "56": 9.64751, + "57": 9.48989, + "58": 9.85502, + "59": 9.59457, + "60": 9.52968, + "61": 9.69589, + "62": 10.01676, + "63": 9.38778, + "64": 9.80211, + "65": 8.95119, + "66": 9.72857, + "67": 9.37577, + "68": 9.80463, + "69": 9.81, + "70": 9.7662, + "71": 9.63135, + "72": 9.5784, + "73": 9.52148, + "74": 8.94976, + "75": 9.43087, + "76": 9.08489, + "77": 10.089, + "78": 9.72754, + "79": 9.37612, + "80": 9.40849, + "81": 9.49766, + "82": 9.71298, + "83": 9.33332, + "84": 9.43928, + "85": 9.63373, + "86": 9.07038, + "87": 9.61245, + "88": 9.78304, + "89": 9.60878, + "90": 9.85164, + "91": 9.34542, + "92": 9.38281, + "93": 9.07319, + "94": 8.81684, + "95": 9.51809, + "96": 9.54033, + "97": 9.34061, + "98": 9.70134, + "99": 8.88786, + "100": 9.43285 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727000.0, + "2": 22925616.0, + "3": 22596924.0, + "4": 23219540.0, + "5": 22714814.0, + "6": 23021786.0, + "7": 22771458.0, + "8": 22926012.0, + "9": 22842856.0, + "10": 22918360.0, + "11": 22500702.0, + "12": 22459866.0, + "13": 22916820.0, + "14": 22389026.0, + "15": 22821260.0, + "16": 22830812.0, + "17": 22818944.0, + "18": 22582240.0, + "19": 22618380.0, + "20": 22694352.0, + "21": 22740180.0, + "22": 22800024.0, + "23": 22540132.0, + "24": 22771492.0, + "25": 22818970.0, + "26": 22546852.0, + "27": 22468504.0, + "28": 22453052.0, + "29": 22529222.0, + "30": 22631432.0, + "31": 22955696.0, + "32": 22585238.0, + "33": 22557676.0, + "34": 22835412.0, + "35": 22788032.0, + "36": 22589678.0, + "37": 22497140.0, + "38": 22896132.0, + "39": 22801314.0, + "40": 22658064.0, + "41": 22659700.0, + "42": 22667816.0, + "43": 22976356.0, + "44": 22746708.0, + "45": 22675272.0, + "46": 22884382.0, + "47": 22634556.0, + "48": 22928080.0, + "49": 22727538.0, + "50": 22905284.0, + "51": 22791326.0, + "52": 22749392.0, + "53": 22925970.0, + "54": 22839434.0, + "55": 22518416.0, + "56": 22877660.0, + "57": 23113304.0, + "58": 22845008.0, + "59": 22715512.0, + "60": 22743058.0, + "61": 22723950.0, + "62": 22673248.0, + "63": 22846074.0, + "64": 22823228.0, + "65": 23060212.0, + "66": 22729902.0, + "67": 22907278.0, + "68": 22610092.0, + "69": 22584360.0, + "70": 22829348.0, + "71": 22749420.0, + "72": 22655446.0, + "73": 22740974.0, + "74": 23048296.0, + "75": 23053922.0, + "76": 22901008.0, + "77": 22272806.0, + "78": 22789370.0, + "79": 22743288.0, + "80": 22706236.0, + "81": 22890976.0, + "82": 22777092.0, + "83": 22839240.0, + "84": 23010352.0, + "85": 22712004.0, + "86": 23103740.0, + "87": 22734788.0, + "88": 22637620.0, + "89": 22499200.0, + "90": 22972420.0, + "91": 22766428.0, + "92": 22808890.0, + "93": 22659888.0, + "94": 22910970.0, + "95": 23048514.0, + "96": 22829470.0, + "97": 22608826.0, + "98": 22763528.0, + "99": 22905754.0, + "100": 23016268.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 719180288.0, + "2": 719180288.0, + "3": 719180288.0, + "4": 719180288.0, + "5": 719180288.0, + "6": 719180288.0, + "7": 719180288.0, + "8": 719180288.0, + "9": 719180288.0, + "10": 719180288.0, + "11": 719180288.0, + "12": 719180288.0, + "13": 719180288.0, + "14": 719180288.0, + "15": 719180288.0, + "16": 719180288.0, + "17": 719180288.0, + "18": 719180288.0, + "19": 719180288.0, + "20": 719180288.0, + "21": 719180288.0, + "22": 719180288.0, + "23": 719180288.0, + "24": 719180288.0, + "25": 719180288.0, + "26": 719180288.0, + "27": 719180288.0, + "28": 719180288.0, + "29": 719180288.0, + "30": 719180288.0, + "31": 719180288.0, + "32": 719180288.0, + "33": 719180288.0, + "34": 719180288.0, + "35": 719180288.0, + "36": 719180288.0, + "37": 719180288.0, + "38": 719180288.0, + "39": 719180288.0, + "40": 719180288.0, + "41": 719180288.0, + "42": 719180288.0, + "43": 719180288.0, + "44": 719180288.0, + "45": 719180288.0, + "46": 719180288.0, + "47": 719180288.0, + "48": 719180288.0, + "49": 719180288.0, + "50": 719180288.0, + "51": 719180288.0, + "52": 719180288.0, + "53": 719180288.0, + "54": 719180288.0, + "55": 719180288.0, + "56": 719180288.0, + "57": 719180288.0, + "58": 719180288.0, + "59": 719180288.0, + "60": 719180288.0, + "61": 719180288.0, + "62": 719180288.0, + "63": 719180288.0, + "64": 719180288.0, + "65": 719180288.0, + "66": 719180288.0, + "67": 719180288.0, + "68": 719180288.0, + "69": 719180288.0, + "70": 719180288.0, + "71": 719180288.0, + "72": 719180288.0, + "73": 719180288.0, + "74": 719180288.0, + "75": 719180288.0, + "76": 719180288.0, + "77": 719180288.0, + "78": 719180288.0, + "79": 719180288.0, + "80": 719180288.0, + "81": 719180288.0, + "82": 719180288.0, + "83": 719180288.0, + "84": 719180288.0, + "85": 719180288.0, + "86": 719180288.0, + "87": 719180288.0, + "88": 719180288.0, + "89": 719180288.0, + "90": 719180288.0, + "91": 719180288.0, + "92": 719180288.0, + "93": 719180288.0, + "94": 719180288.0, + "95": 719180288.0, + "96": 719180288.0, + "97": 719180288.0, + "98": 719180288.0, + "99": 719180288.0, + "100": 719180288.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910562816.0, + "2": 2195420160.0, + "3": 2195420160.0, + "4": 2195420160.0, + "5": 2195420160.0, + "6": 2195420160.0, + "7": 2195420160.0, + "8": 2195420160.0, + "9": 2195420160.0, + "10": 2195420160.0, + "11": 2195420160.0, + "12": 2195420160.0, + "13": 2195420160.0, + "14": 2195420160.0, + "15": 2195420160.0, + "16": 2195420160.0, + "17": 2195420160.0, + "18": 2195420160.0, + "19": 2195420160.0, + "20": 2195420160.0, + "21": 2195420160.0, + "22": 2195420160.0, + "23": 2195420160.0, + "24": 2195420160.0, + "25": 2195420160.0, + "26": 2195420160.0, + "27": 2195420160.0, + "28": 2195420160.0, + "29": 2195420160.0, + "30": 2195420160.0, + "31": 2195420160.0, + "32": 2195420160.0, + "33": 2195420160.0, + "34": 2195420160.0, + "35": 2195420160.0, + "36": 2195420160.0, + "37": 2195420160.0, + "38": 2195420160.0, + "39": 2195420160.0, + "40": 2195420160.0, + "41": 2195420160.0, + "42": 2195420160.0, + "43": 2195420160.0, + "44": 2195420160.0, + "45": 2195420160.0, + "46": 2195420160.0, + "47": 2195420160.0, + "48": 2195420160.0, + "49": 2195420160.0, + "50": 2195420160.0, + "51": 2195420160.0, + "52": 2195420160.0, + "53": 2195420160.0, + "54": 2195420160.0, + "55": 2195420160.0, + "56": 2195420160.0, + "57": 2195420160.0, + "58": 2195420160.0, + "59": 2195420160.0, + "60": 2195420160.0, + "61": 2195420160.0, + "62": 2195420160.0, + "63": 2195420160.0, + "64": 2195420160.0, + "65": 2195420160.0, + "66": 2195420160.0, + "67": 2195420160.0, + "68": 2195420160.0, + "69": 2195420160.0, + "70": 2195420160.0, + "71": 2195420160.0, + "72": 2195420160.0, + "73": 2195420160.0, + "74": 2195420160.0, + "75": 2195420160.0, + "76": 2195420160.0, + "77": 2195420160.0, + "78": 2195420160.0, + "79": 2195420160.0, + "80": 2195420160.0, + "81": 2195420160.0, + "82": 2195420160.0, + "83": 2195420160.0, + "84": 2195420160.0, + "85": 2195420160.0, + "86": 2195420160.0, + "87": 2195420160.0, + "88": 2195420160.0, + "89": 2195420160.0, + "90": 2195420160.0, + "91": 2195420160.0, + "92": 2195420160.0, + "93": 2195420160.0, + "94": 2195420160.0, + "95": 2195420160.0, + "96": 2195420160.0, + "97": 2195420160.0, + "98": 2195420160.0, + "99": 2195420160.0, + "100": 2195420160.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.03051, + "2": 0.1884, + "3": 0.27987, + "4": 0.15687, + "5": 0.15614, + "6": 0.1563, + "7": 0.15622, + "8": 0.1563, + "9": 0.15473, + "10": 0.15615, + "11": 0.15528, + "12": 0.15754, + "13": 0.15661, + "14": 0.15677, + "15": 0.15572, + "16": 0.15582, + "17": 0.1571, + "18": 0.15664, + "19": 0.1556, + "20": 0.15592, + "21": 0.15572, + "22": 0.15642, + "23": 0.15643, + "24": 0.15743, + "25": 0.15666, + "26": 0.15552, + "27": 0.15475, + "28": 0.15586, + "29": 0.15603, + "30": 0.15463, + "31": 0.15712, + "32": 0.15442, + "33": 0.15543, + "34": 0.1557, + "35": 0.15682, + "36": 0.15539, + "37": 0.1553, + "38": 0.15578, + "39": 0.15667, + "40": 0.15715, + "41": 0.15704, + "42": 0.1566, + "43": 0.15655, + "44": 0.15629, + "45": 0.15584, + "46": 0.15734, + "47": 0.15735, + "48": 0.15572, + "49": 0.15706, + "50": 0.15561, + "51": 0.16957, + "52": 0.1587, + "53": 0.16014, + "54": 0.15805, + "55": 0.1578, + "56": 0.15801, + "57": 0.15813, + "58": 0.1574, + "59": 0.15781, + "60": 0.15923, + "61": 0.15655, + "62": 0.15633, + "63": 0.15583, + "64": 0.15734, + "65": 0.15761, + "66": 0.15822, + "67": 0.15755, + "68": 0.15815, + "69": 0.15816, + "70": 0.15813, + "71": 0.15747, + "72": 0.1574, + "73": 0.15783, + "74": 0.15766, + "75": 0.15527, + "76": 0.15579, + "77": 0.15483, + "78": 0.15482, + "79": 0.15404, + "80": 0.15431, + "81": 0.154, + "82": 0.15512, + "83": 0.15513, + "84": 0.15371, + "85": 0.15488, + "86": 0.15465, + "87": 0.15412, + "88": 0.15403, + "89": 0.15487, + "90": 0.15518, + "91": 0.15549, + "92": 0.154, + "93": 0.15405, + "94": 0.15438, + "95": 0.15444, + "96": 0.1534, + "97": 0.15487, + "98": 0.15398, + "99": 0.15434, + "100": 0.15391 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json index 11578ee6c3e..307cec2659c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.90105, - "5": 10.89686, - "10": 10.88269, - "15": 10.83975, - "20": 10.74037, - "25": 10.57931, - "30": 10.37738, - "35": 10.29033, - "40": 10.1156, - "45": 9.85639, - "50": 9.93378, - "55": 9.87553, - "60": 9.528, - "65": 8.95041, - "70": 9.76634, - "75": 9.4304, - "80": 9.40916, - "85": 9.63365, - "90": 9.8516, - "95": 9.51874, - "100": 9.43257 + "1": 10.90314, + "2": 10.88897, + "3": 10.89725, + "4": 10.88248, + "5": 10.89447, + "6": 10.91226, + "7": 10.89775, + "8": 10.88643, + "9": 10.89227, + "10": 10.8849, + "11": 10.91413, + "12": 10.88666, + "13": 10.89283, + "14": 10.90486, + "15": 10.83878, + "16": 10.84927, + "17": 10.83248, + "18": 10.83371, + "19": 10.83359, + "20": 10.74074, + "21": 10.7043, + "22": 10.59893, + "23": 10.72222, + "24": 10.60745, + "25": 10.57467, + "26": 10.62999, + "27": 10.62364, + "28": 10.57229, + "29": 10.6073, + "30": 10.37766, + "31": 10.15362, + "32": 10.47609, + "33": 10.48062, + "34": 10.24216, + "35": 10.29035, + "36": 10.25955, + "37": 10.36145, + "38": 10.21396, + "39": 10.44502, + "40": 10.11492, + "41": 10.1605, + "42": 10.23468, + "43": 9.85032, + "44": 9.97764, + "45": 9.85681, + "46": 9.8307, + "47": 10.17976, + "48": 9.85811, + "49": 9.54378, + "50": 9.93469, + "51": 9.86793, + "52": 9.76274, + "53": 10.10895, + "54": 9.95538, + "55": 9.8756, + "56": 9.64751, + "57": 9.48989, + "58": 9.85502, + "59": 9.59457, + "60": 9.52968, + "61": 9.69589, + "62": 10.01676, + "63": 9.38778, + "64": 9.80211, + "65": 8.95119, + "66": 9.72857, + "67": 9.37577, + "68": 9.80463, + "69": 9.81, + "70": 9.7662, + "71": 9.63135, + "72": 9.5784, + "73": 9.52148, + "74": 8.94976, + "75": 9.43087, + "76": 9.08489, + "77": 10.089, + "78": 9.72754, + "79": 9.37612, + "80": 9.40849, + "81": 9.49766, + "82": 9.71298, + "83": 9.33332, + "84": 9.43928, + "85": 9.63373, + "86": 9.07038, + "87": 9.61245, + "88": 9.78304, + "89": 9.60878, + "90": 9.85164, + "91": 9.34542, + "92": 9.38281, + "93": 9.07319, + "94": 8.81684, + "95": 9.51809, + "96": 9.54033, + "97": 9.34061, + "98": 9.70134, + "99": 8.88786, + "100": 9.43285 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 22727086.0, - "5": 22714736.0, - "10": 22918912.0, - "15": 22821334.0, - "20": 22694008.0, - "25": 22818912.0, - "30": 22631266.0, - "35": 22787944.0, - "40": 22658196.0, - "45": 22675296.0, - "50": 22904808.0, - "55": 22518754.0, - "60": 22743504.0, - "65": 23060460.0, - "70": 22829772.0, - "75": 23054048.0, - "80": 22706308.0, - "85": 22712054.0, - "90": 22972128.0, - "95": 23048144.0, - "100": 23015824.0 + "1": 22727000.0, + "2": 22925616.0, + "3": 22596924.0, + "4": 23219540.0, + "5": 22714814.0, + "6": 23021786.0, + "7": 22771458.0, + "8": 22926012.0, + "9": 22842856.0, + "10": 22918360.0, + "11": 22500702.0, + "12": 22459866.0, + "13": 22916820.0, + "14": 22389026.0, + "15": 22821260.0, + "16": 22830812.0, + "17": 22818944.0, + "18": 22582240.0, + "19": 22618380.0, + "20": 22694352.0, + "21": 22740180.0, + "22": 22800024.0, + "23": 22540132.0, + "24": 22771492.0, + "25": 22818970.0, + "26": 22546852.0, + "27": 22468504.0, + "28": 22453052.0, + "29": 22529222.0, + "30": 22631432.0, + "31": 22955696.0, + "32": 22585238.0, + "33": 22557676.0, + "34": 22835412.0, + "35": 22788032.0, + "36": 22589678.0, + "37": 22497140.0, + "38": 22896132.0, + "39": 22801314.0, + "40": 22658064.0, + "41": 22659700.0, + "42": 22667816.0, + "43": 22976356.0, + "44": 22746708.0, + "45": 22675272.0, + "46": 22884382.0, + "47": 22634556.0, + "48": 22928080.0, + "49": 22727538.0, + "50": 22905284.0, + "51": 22791326.0, + "52": 22749392.0, + "53": 22925970.0, + "54": 22839434.0, + "55": 22518416.0, + "56": 22877660.0, + "57": 23113304.0, + "58": 22845008.0, + "59": 22715512.0, + "60": 22743058.0, + "61": 22723950.0, + "62": 22673248.0, + "63": 22846074.0, + "64": 22823228.0, + "65": 23060212.0, + "66": 22729902.0, + "67": 22907278.0, + "68": 22610092.0, + "69": 22584360.0, + "70": 22829348.0, + "71": 22749420.0, + "72": 22655446.0, + "73": 22740974.0, + "74": 23048296.0, + "75": 23053922.0, + "76": 22901008.0, + "77": 22272806.0, + "78": 22789370.0, + "79": 22743288.0, + "80": 22706236.0, + "81": 22890976.0, + "82": 22777092.0, + "83": 22839240.0, + "84": 23010352.0, + "85": 22712004.0, + "86": 23103740.0, + "87": 22734788.0, + "88": 22637620.0, + "89": 22499200.0, + "90": 22972420.0, + "91": 22766428.0, + "92": 22808890.0, + "93": 22659888.0, + "94": 22910970.0, + "95": 23048514.0, + "96": 22829470.0, + "97": 22608826.0, + "98": 22763528.0, + "99": 22905754.0, + "100": 23016268.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 717082624.0, - "5": 717082624.0, - "10": 717082624.0, - "15": 717082624.0, - "20": 717082624.0, - "25": 717082624.0, - "30": 717082624.0, - "35": 717082624.0, - "40": 717082624.0, - "45": 717082624.0, - "50": 717082624.0, - "55": 717082624.0, - "60": 717082624.0, - "65": 717082624.0, - "70": 717082624.0, - "75": 717082624.0, - "80": 717082624.0, - "85": 717082624.0, - "90": 717082624.0, - "95": 717082624.0, - "100": 717082624.0 + "1": 719180288.0, + "2": 719180288.0, + "3": 719180288.0, + "4": 719180288.0, + "5": 719180288.0, + "6": 719180288.0, + "7": 719180288.0, + "8": 719180288.0, + "9": 719180288.0, + "10": 719180288.0, + "11": 719180288.0, + "12": 719180288.0, + "13": 719180288.0, + "14": 719180288.0, + "15": 719180288.0, + "16": 719180288.0, + "17": 719180288.0, + "18": 719180288.0, + "19": 719180288.0, + "20": 719180288.0, + "21": 719180288.0, + "22": 719180288.0, + "23": 719180288.0, + "24": 719180288.0, + "25": 719180288.0, + "26": 719180288.0, + "27": 719180288.0, + "28": 719180288.0, + "29": 719180288.0, + "30": 719180288.0, + "31": 719180288.0, + "32": 719180288.0, + "33": 719180288.0, + "34": 719180288.0, + "35": 719180288.0, + "36": 719180288.0, + "37": 719180288.0, + "38": 719180288.0, + "39": 719180288.0, + "40": 719180288.0, + "41": 719180288.0, + "42": 719180288.0, + "43": 719180288.0, + "44": 719180288.0, + "45": 719180288.0, + "46": 719180288.0, + "47": 719180288.0, + "48": 719180288.0, + "49": 719180288.0, + "50": 719180288.0, + "51": 719180288.0, + "52": 719180288.0, + "53": 719180288.0, + "54": 719180288.0, + "55": 719180288.0, + "56": 719180288.0, + "57": 719180288.0, + "58": 719180288.0, + "59": 719180288.0, + "60": 719180288.0, + "61": 719180288.0, + "62": 719180288.0, + "63": 719180288.0, + "64": 719180288.0, + "65": 719180288.0, + "66": 719180288.0, + "67": 719180288.0, + "68": 719180288.0, + "69": 719180288.0, + "70": 719180288.0, + "71": 719180288.0, + "72": 719180288.0, + "73": 719180288.0, + "74": 719180288.0, + "75": 719180288.0, + "76": 719180288.0, + "77": 719180288.0, + "78": 719180288.0, + "79": 719180288.0, + "80": 719180288.0, + "81": 719180288.0, + "82": 719180288.0, + "83": 719180288.0, + "84": 719180288.0, + "85": 719180288.0, + "86": 719180288.0, + "87": 719180288.0, + "88": 719180288.0, + "89": 719180288.0, + "90": 719180288.0, + "91": 719180288.0, + "92": 719180288.0, + "93": 719180288.0, + "94": 719180288.0, + "95": 719180288.0, + "96": 719180288.0, + "97": 719180288.0, + "98": 719180288.0, + "99": 719180288.0, + "100": 719180288.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 2399852544.0, - "5": 2683661312.0, - "10": 2683661312.0, - "15": 2683661312.0, - "20": 2683661312.0, - "25": 2683661312.0, - "30": 2683661312.0, - "35": 2683661312.0, - "40": 2683661312.0, - "45": 2683661312.0, - "50": 2683661312.0, - "55": 2683661312.0, - "60": 2683661312.0, - "65": 2683661312.0, - "70": 2683661312.0, - "75": 2683661312.0, - "80": 2683661312.0, - "85": 2683661312.0, - "90": 2683661312.0, - "95": 2683661312.0, - "100": 2683661312.0 + "1": 1910562816.0, + "2": 2195420160.0, + "3": 2195420160.0, + "4": 2195420160.0, + "5": 2195420160.0, + "6": 2195420160.0, + "7": 2195420160.0, + "8": 2195420160.0, + "9": 2195420160.0, + "10": 2195420160.0, + "11": 2195420160.0, + "12": 2195420160.0, + "13": 2195420160.0, + "14": 2195420160.0, + "15": 2195420160.0, + "16": 2195420160.0, + "17": 2195420160.0, + "18": 2195420160.0, + "19": 2195420160.0, + "20": 2195420160.0, + "21": 2195420160.0, + "22": 2195420160.0, + "23": 2195420160.0, + "24": 2195420160.0, + "25": 2195420160.0, + "26": 2195420160.0, + "27": 2195420160.0, + "28": 2195420160.0, + "29": 2195420160.0, + "30": 2195420160.0, + "31": 2195420160.0, + "32": 2195420160.0, + "33": 2195420160.0, + "34": 2195420160.0, + "35": 2195420160.0, + "36": 2195420160.0, + "37": 2195420160.0, + "38": 2195420160.0, + "39": 2195420160.0, + "40": 2195420160.0, + "41": 2195420160.0, + "42": 2195420160.0, + "43": 2195420160.0, + "44": 2195420160.0, + "45": 2195420160.0, + "46": 2195420160.0, + "47": 2195420160.0, + "48": 2195420160.0, + "49": 2195420160.0, + "50": 2195420160.0, + "51": 2195420160.0, + "52": 2195420160.0, + "53": 2195420160.0, + "54": 2195420160.0, + "55": 2195420160.0, + "56": 2195420160.0, + "57": 2195420160.0, + "58": 2195420160.0, + "59": 2195420160.0, + "60": 2195420160.0, + "61": 2195420160.0, + "62": 2195420160.0, + "63": 2195420160.0, + "64": 2195420160.0, + "65": 2195420160.0, + "66": 2195420160.0, + "67": 2195420160.0, + "68": 2195420160.0, + "69": 2195420160.0, + "70": 2195420160.0, + "71": 2195420160.0, + "72": 2195420160.0, + "73": 2195420160.0, + "74": 2195420160.0, + "75": 2195420160.0, + "76": 2195420160.0, + "77": 2195420160.0, + "78": 2195420160.0, + "79": 2195420160.0, + "80": 2195420160.0, + "81": 2195420160.0, + "82": 2195420160.0, + "83": 2195420160.0, + "84": 2195420160.0, + "85": 2195420160.0, + "86": 2195420160.0, + "87": 2195420160.0, + "88": 2195420160.0, + "89": 2195420160.0, + "90": 2195420160.0, + "91": 2195420160.0, + "92": 2195420160.0, + "93": 2195420160.0, + "94": 2195420160.0, + "95": 2195420160.0, + "96": 2195420160.0, + "97": 2195420160.0, + "98": 2195420160.0, + "99": 2195420160.0, + "100": 2195420160.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 18.12809, - "5": 0.18057, - "10": 0.17706, - "15": 0.17756, - "20": 0.17731, - "25": 0.17728, - "30": 0.1768, - "35": 0.17506, - "40": 0.1753, - "45": 0.17703, - "50": 0.17481, - "55": 0.17327, - "60": 0.17184, - "65": 0.17221, - "70": 0.17198, - "75": 0.1719, - "80": 0.17197, - "85": 0.1726, - "90": 0.17157, - "95": 0.17135, - "100": 0.17214 + "1": 10.1537, + "2": 0.18498, + "3": 0.16024, + "4": 0.16059, + "5": 0.16002, + "6": 0.16103, + "7": 0.1591, + "8": 0.15912, + "9": 0.15909, + "10": 0.1574, + "11": 0.15721, + "12": 0.15764, + "13": 0.16009, + "14": 0.16035, + "15": 0.15973, + "16": 0.15641, + "17": 0.15673, + "18": 0.1565, + "19": 0.15684, + "20": 0.15713, + "21": 0.15762, + "22": 0.15859, + "23": 0.15877, + "24": 0.15973, + "25": 0.15946, + "26": 0.15909, + "27": 0.15855, + "28": 0.15876, + "29": 0.15921, + "30": 0.16148, + "31": 0.15991, + "32": 0.1576, + "33": 0.15829, + "34": 0.15886, + "35": 0.15948, + "36": 0.15819, + "37": 0.15886, + "38": 0.15896, + "39": 0.16029, + "40": 0.15802, + "41": 0.16038, + "42": 0.15965, + "43": 0.15985, + "44": 0.15882, + "45": 0.16056, + "46": 0.1592, + "47": 0.20747, + "48": 0.16124, + "49": 0.16012, + "50": 0.15759, + "51": 0.16615, + "52": 0.15685, + "53": 0.15965, + "54": 0.15787, + "55": 0.15762, + "56": 0.15748, + "57": 0.15807, + "58": 0.15831, + "59": 0.15671, + "60": 0.15765, + "61": 0.15997, + "62": 0.15756, + "63": 0.15822, + "64": 0.15898, + "65": 0.15778, + "66": 0.15853, + "67": 0.15855, + "68": 0.15784, + "69": 0.15777, + "70": 0.15791, + "71": 0.15907, + "72": 0.15986, + "73": 0.15727, + "74": 0.15842, + "75": 0.15738, + "76": 0.15786, + "77": 0.15749, + "78": 0.15761, + "79": 0.15838, + "80": 0.15955, + "81": 0.15796, + "82": 0.15816, + "83": 0.15953, + "84": 0.15849, + "85": 0.15905, + "86": 0.15852, + "87": 0.15827, + "88": 0.15773, + "89": 0.15778, + "90": 0.15679, + "91": 0.1583, + "92": 0.15749, + "93": 0.15843, + "94": 0.15878, + "95": 0.15805, + "96": 0.1588, + "97": 0.15983, + "98": 0.16098, + "99": 0.16131, + "100": 0.15935 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json new file mode 100644 index 00000000000..eee2d1c3d29 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79513, + "16": 10.7958, + "17": 10.76832, + "18": 10.8109, + "19": 10.79887, + "20": 10.69131, + "21": 10.6801, + "22": 10.52152, + "23": 10.7071, + "24": 10.57678, + "25": 10.52316, + "26": 10.59563, + "27": 10.58607, + "28": 10.56175, + "29": 10.56945, + "30": 10.34623, + "31": 10.10035, + "32": 10.45432, + "33": 10.44591, + "34": 10.2072, + "35": 10.26185, + "36": 10.21228, + "37": 10.32449, + "38": 10.16803, + "39": 10.38353, + "40": 10.07219, + "41": 10.13754, + "42": 10.19755, + "43": 9.81134, + "44": 9.93287, + "45": 9.80998, + "46": 9.80859, + "47": 10.12583, + "48": 9.82132, + "49": 9.50738, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1894.0, + "16": 1731.0, + "17": 1848.0, + "18": 1705.0, + "19": 1686.0, + "20": 1620.0, + "21": 1840.0, + "22": 1764.0, + "23": 1937.0, + "24": 1620.0, + "25": 1728.0, + "26": 1727.0, + "27": 1821.0, + "28": 2042.0, + "29": 2029.0, + "30": 1825.0, + "31": 1594.0, + "32": 1903.0, + "33": 2041.0, + "34": 1895.0, + "35": 1908.0, + "36": 1906.0, + "37": 2224.0, + "38": 2150.0, + "39": 2327.0, + "40": 2074.0, + "41": 2314.0, + "42": 2230.0, + "43": 1920.0, + "44": 2115.0, + "45": 1962.0, + "46": 2287.0, + "47": 2481.0, + "48": 2407.0, + "49": 2270.0, + "50": 2349.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3837453824.0, + "2": 4119165440.0, + "3": 4119165440.0, + "4": 4119165440.0, + "5": 4119165440.0, + "6": 4119165440.0, + "7": 4119165440.0, + "8": 4119165440.0, + "9": 4119165440.0, + "10": 4119165440.0, + "11": 4119165440.0, + "12": 4119165440.0, + "13": 4119165440.0, + "14": 4119165440.0, + "15": 4119165440.0, + "16": 4119165440.0, + "17": 4119165440.0, + "18": 4119165440.0, + "19": 4119165440.0, + "20": 4119165440.0, + "21": 4119165440.0, + "22": 4119165440.0, + "23": 4119165440.0, + "24": 4119165440.0, + "25": 4119165440.0, + "26": 4119165440.0, + "27": 4119165440.0, + "28": 4119165440.0, + "29": 4119165440.0, + "30": 4119165440.0, + "31": 4119165440.0, + "32": 4119165440.0, + "33": 4119165440.0, + "34": 4119165440.0, + "35": 4119165440.0, + "36": 4119165440.0, + "37": 4119165440.0, + "38": 4119165440.0, + "39": 4119165440.0, + "40": 4119165440.0, + "41": 4119165440.0, + "42": 4119165440.0, + "43": 4119165440.0, + "44": 4119165440.0, + "45": 4119165440.0, + "46": 4119165440.0, + "47": 4119165440.0, + "48": 4119165440.0, + "49": 4119165440.0, + "50": 4119165440.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.00614, + "2": 0.18804, + "3": 0.28065, + "4": 0.14064, + "5": 0.1388, + "6": 0.13824, + "7": 0.13862, + "8": 0.13843, + "9": 0.14064, + "10": 0.13568, + "11": 0.13353, + "12": 0.13327, + "13": 0.13418, + "14": 0.13368, + "15": 0.13399, + "16": 0.13326, + "17": 0.13409, + "18": 0.13281, + "19": 0.13303, + "20": 0.13395, + "21": 0.13357, + "22": 0.13388, + "23": 0.13403, + "24": 0.1333, + "25": 0.13242, + "26": 0.13302, + "27": 0.134, + "28": 0.13304, + "29": 0.13302, + "30": 0.13398, + "31": 0.13424, + "32": 0.13315, + "33": 0.13365, + "34": 0.13391, + "35": 0.13392, + "36": 0.13316, + "37": 0.13254, + "38": 0.13292, + "39": 0.1333, + "40": 0.13401, + "41": 0.13408, + "42": 0.13349, + "43": 0.13469, + "44": 0.13282, + "45": 0.1344, + "46": 0.13536, + "47": 0.13291, + "48": 0.13374, + "49": 0.13338, + "50": 0.13336 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json index 8faccf70250..4bf73c8b005 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69211, "25": 10.52412, "30": 10.34552, "35": 10.26239, "40": 10.07241, "45": 9.81101, "50": 9.88422}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1601.0, "25": 1635.0, "30": 1936.0, "35": 1908.0, "40": 2100.0, "45": 2098.0, "50": 2333.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 14.9121, "5": 0.1731, "10": 0.17256, "15": 0.1722, "20": 0.17555, "25": 0.17245, "30": 0.17067, "35": 0.17091, "40": 0.17274, "45": 0.17151, "50": 0.17108}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79513, + "16": 10.7958, + "17": 10.76832, + "18": 10.8109, + "19": 10.79887, + "20": 10.69131, + "21": 10.6801, + "22": 10.52152, + "23": 10.7071, + "24": 10.57678, + "25": 10.52316, + "26": 10.59563, + "27": 10.58607, + "28": 10.56175, + "29": 10.56945, + "30": 10.34623, + "31": 10.10035, + "32": 10.45432, + "33": 10.44591, + "34": 10.2072, + "35": 10.26185, + "36": 10.21228, + "37": 10.32449, + "38": 10.16803, + "39": 10.38353, + "40": 10.07219, + "41": 10.13754, + "42": 10.19755, + "43": 9.81134, + "44": 9.93287, + "45": 9.80998, + "46": 9.80859, + "47": 10.12583, + "48": 9.82132, + "49": 9.50738, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1894.0, + "16": 1731.0, + "17": 1848.0, + "18": 1705.0, + "19": 1686.0, + "20": 1620.0, + "21": 1840.0, + "22": 1764.0, + "23": 1937.0, + "24": 1620.0, + "25": 1728.0, + "26": 1727.0, + "27": 1821.0, + "28": 2042.0, + "29": 2029.0, + "30": 1825.0, + "31": 1594.0, + "32": 1903.0, + "33": 2041.0, + "34": 1895.0, + "35": 1908.0, + "36": 1906.0, + "37": 2224.0, + "38": 2150.0, + "39": 2327.0, + "40": 2074.0, + "41": 2314.0, + "42": 2230.0, + "43": 1920.0, + "44": 2115.0, + "45": 1962.0, + "46": 2287.0, + "47": 2481.0, + "48": 2407.0, + "49": 2270.0, + "50": 2349.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3837453824.0, + "2": 4119165440.0, + "3": 4119165440.0, + "4": 4119165440.0, + "5": 4119165440.0, + "6": 4119165440.0, + "7": 4119165440.0, + "8": 4119165440.0, + "9": 4119165440.0, + "10": 4119165440.0, + "11": 4119165440.0, + "12": 4119165440.0, + "13": 4119165440.0, + "14": 4119165440.0, + "15": 4119165440.0, + "16": 4119165440.0, + "17": 4119165440.0, + "18": 4119165440.0, + "19": 4119165440.0, + "20": 4119165440.0, + "21": 4119165440.0, + "22": 4119165440.0, + "23": 4119165440.0, + "24": 4119165440.0, + "25": 4119165440.0, + "26": 4119165440.0, + "27": 4119165440.0, + "28": 4119165440.0, + "29": 4119165440.0, + "30": 4119165440.0, + "31": 4119165440.0, + "32": 4119165440.0, + "33": 4119165440.0, + "34": 4119165440.0, + "35": 4119165440.0, + "36": 4119165440.0, + "37": 4119165440.0, + "38": 4119165440.0, + "39": 4119165440.0, + "40": 4119165440.0, + "41": 4119165440.0, + "42": 4119165440.0, + "43": 4119165440.0, + "44": 4119165440.0, + "45": 4119165440.0, + "46": 4119165440.0, + "47": 4119165440.0, + "48": 4119165440.0, + "49": 4119165440.0, + "50": 4119165440.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.25479, + "2": 0.18004, + "3": 0.15444, + "4": 0.15284, + "5": 0.15391, + "6": 0.14333, + "7": 0.14244, + "8": 0.13997, + "9": 0.14112, + "10": 0.13863, + "11": 0.13707, + "12": 0.13575, + "13": 0.13558, + "14": 0.13535, + "15": 0.13556, + "16": 0.13648, + "17": 0.13495, + "18": 0.1343, + "19": 0.13442, + "20": 0.13441, + "21": 0.1344, + "22": 0.13478, + "23": 0.13473, + "24": 0.13476, + "25": 0.13536, + "26": 0.13345, + "27": 0.1342, + "28": 0.13421, + "29": 0.13479, + "30": 0.13378, + "31": 0.13418, + "32": 0.13411, + "33": 0.13351, + "34": 0.13374, + "35": 0.13406, + "36": 0.13396, + "37": 0.13435, + "38": 0.13356, + "39": 0.13367, + "40": 0.13361, + "41": 0.13454, + "42": 0.13463, + "43": 0.13524, + "44": 0.13356, + "45": 0.13403, + "46": 0.1347, + "47": 0.13379, + "48": 0.1343, + "49": 0.13391, + "50": 0.13371 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..1553ecdd445 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79514, + "16": 10.79576, + "17": 10.76832, + "18": 10.81092, + "19": 10.79889, + "20": 10.69131, + "21": 10.68008, + "22": 10.52146, + "23": 10.70713, + "24": 10.57677, + "25": 10.52315, + "26": 10.59564, + "27": 10.5861, + "28": 10.56176, + "29": 10.56942, + "30": 10.34624, + "31": 10.10032, + "32": 10.45433, + "33": 10.44592, + "34": 10.20725, + "35": 10.26186, + "36": 10.21223, + "37": 10.32453, + "38": 10.16801, + "39": 10.38354, + "40": 10.07222, + "41": 10.13752, + "42": 10.19756, + "43": 9.81134, + "44": 9.93285, + "45": 9.81001, + "46": 9.80858, + "47": 10.12582, + "48": 9.82129, + "49": 9.50739, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1839.0, + "16": 1776.0, + "17": 1750.0, + "18": 1612.0, + "19": 1764.0, + "20": 1649.0, + "21": 1854.0, + "22": 1750.0, + "23": 1909.0, + "24": 1616.0, + "25": 1654.0, + "26": 1755.0, + "27": 1860.0, + "28": 2042.0, + "29": 1953.0, + "30": 1905.0, + "31": 1684.0, + "32": 1831.0, + "33": 2101.0, + "34": 1769.0, + "35": 1915.0, + "36": 1885.0, + "37": 2324.0, + "38": 2169.0, + "39": 2300.0, + "40": 2069.0, + "41": 2353.0, + "42": 2236.0, + "43": 1978.0, + "44": 2022.0, + "45": 2103.0, + "46": 2292.0, + "47": 2413.0, + "48": 2305.0, + "49": 2218.0, + "50": 2321.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 523371008.0, + "2": 523371008.0, + "3": 523371008.0, + "4": 523371008.0, + "5": 523371008.0, + "6": 523371008.0, + "7": 523371008.0, + "8": 523371008.0, + "9": 523371008.0, + "10": 523371008.0, + "11": 523371008.0, + "12": 523371008.0, + "13": 523371008.0, + "14": 523371008.0, + "15": 523371008.0, + "16": 523371008.0, + "17": 523371008.0, + "18": 523371008.0, + "19": 523371008.0, + "20": 523371008.0, + "21": 523371008.0, + "22": 523371008.0, + "23": 523371008.0, + "24": 523371008.0, + "25": 523371008.0, + "26": 523371008.0, + "27": 523371008.0, + "28": 523371008.0, + "29": 523371008.0, + "30": 523371008.0, + "31": 523371008.0, + "32": 523371008.0, + "33": 523371008.0, + "34": 523371008.0, + "35": 523371008.0, + "36": 523371008.0, + "37": 523371008.0, + "38": 523371008.0, + "39": 523371008.0, + "40": 523371008.0, + "41": 523371008.0, + "42": 523371008.0, + "43": 523371008.0, + "44": 523371008.0, + "45": 523371008.0, + "46": 523371008.0, + "47": 523371008.0, + "48": 523371008.0, + "49": 523371008.0, + "50": 523371008.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3768846848.0, + "2": 3913263616.0, + "3": 3913263616.0, + "4": 3913263616.0, + "5": 3913263616.0, + "6": 3913263616.0, + "7": 3913263616.0, + "8": 3913263616.0, + "9": 3913263616.0, + "10": 3913263616.0, + "11": 3913263616.0, + "12": 3913263616.0, + "13": 3913263616.0, + "14": 3913263616.0, + "15": 3913263616.0, + "16": 3913263616.0, + "17": 3913263616.0, + "18": 3913263616.0, + "19": 3913263616.0, + "20": 3913263616.0, + "21": 3913263616.0, + "22": 3913263616.0, + "23": 3913263616.0, + "24": 3913263616.0, + "25": 3913263616.0, + "26": 3913263616.0, + "27": 3913263616.0, + "28": 3913263616.0, + "29": 3913263616.0, + "30": 3913263616.0, + "31": 3913263616.0, + "32": 3913263616.0, + "33": 3913263616.0, + "34": 3913263616.0, + "35": 3913263616.0, + "36": 3913263616.0, + "37": 3913263616.0, + "38": 3913263616.0, + "39": 3913263616.0, + "40": 3913263616.0, + "41": 3913263616.0, + "42": 3913263616.0, + "43": 3913263616.0, + "44": 3913263616.0, + "45": 3913263616.0, + "46": 3913263616.0, + "47": 3913263616.0, + "48": 3913263616.0, + "49": 3913263616.0, + "50": 3913263616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.01582, + "2": 0.17662, + "3": 0.15135, + "4": 0.14999, + "5": 0.14829, + "6": 0.14621, + "7": 0.14816, + "8": 0.14731, + "9": 0.14766, + "10": 0.14515, + "11": 0.30054, + "12": 0.14534, + "13": 0.14429, + "14": 0.14592, + "15": 0.14632, + "16": 0.14618, + "17": 0.14537, + "18": 0.14666, + "19": 0.14384, + "20": 0.14453, + "21": 0.30388, + "22": 0.14466, + "23": 0.14511, + "24": 0.14435, + "25": 0.14401, + "26": 0.14328, + "27": 0.14376, + "28": 0.14434, + "29": 0.14386, + "30": 0.14418, + "31": 0.30313, + "32": 0.14394, + "33": 0.14406, + "34": 0.14377, + "35": 0.14417, + "36": 0.14415, + "37": 0.14393, + "38": 0.14577, + "39": 0.14494, + "40": 0.14489, + "41": 0.30235, + "42": 0.14494, + "43": 0.1472, + "44": 0.14577, + "45": 0.14497, + "46": 0.14619, + "47": 0.14474, + "48": 0.14551, + "49": 0.14554, + "50": 0.14507 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json index dc393d0dffc..35ef87a5085 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.7923, "20": 10.69213, "25": 10.5241, "30": 10.34556, "35": 10.26241, "40": 10.07237, "45": 9.811, "50": 9.88419}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1850.0, "20": 1699.0, "25": 1614.0, "30": 1905.0, "35": 1933.0, "40": 2169.0, "45": 2101.0, "50": 2421.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 523004928.0, "5": 523004928.0, "10": 523004928.0, "15": 523004928.0, "20": 523004928.0, "25": 523004928.0, "30": 523004928.0, "35": 523004928.0, "40": 523004928.0, "45": 523004928.0, "50": 523004928.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3768873984.0, "5": 3912766464.0, "10": 3912766464.0, "15": 3912766464.0, "20": 3912766464.0, "25": 3912766464.0, "30": 3912766464.0, "35": 3912766464.0, "40": 3912766464.0, "45": 3912766464.0, "50": 3912766464.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 18.88705, "5": 0.16956, "10": 0.17448, "15": 0.16853, "20": 0.1715, "25": 0.17071, "30": 0.17343, "35": 0.17213, "40": 0.1719, "45": 0.17357, "50": 0.17228}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79514, + "16": 10.79576, + "17": 10.76832, + "18": 10.81092, + "19": 10.79889, + "20": 10.69131, + "21": 10.68008, + "22": 10.52146, + "23": 10.70713, + "24": 10.57677, + "25": 10.52315, + "26": 10.59564, + "27": 10.5861, + "28": 10.56176, + "29": 10.56942, + "30": 10.34624, + "31": 10.10032, + "32": 10.45433, + "33": 10.44592, + "34": 10.20725, + "35": 10.26186, + "36": 10.21223, + "37": 10.32453, + "38": 10.16801, + "39": 10.38354, + "40": 10.07222, + "41": 10.13752, + "42": 10.19756, + "43": 9.81134, + "44": 9.93285, + "45": 9.81001, + "46": 9.80858, + "47": 10.12582, + "48": 9.82129, + "49": 9.50739, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1839.0, + "16": 1776.0, + "17": 1750.0, + "18": 1612.0, + "19": 1764.0, + "20": 1649.0, + "21": 1854.0, + "22": 1750.0, + "23": 1909.0, + "24": 1616.0, + "25": 1654.0, + "26": 1755.0, + "27": 1860.0, + "28": 2042.0, + "29": 1953.0, + "30": 1905.0, + "31": 1684.0, + "32": 1831.0, + "33": 2101.0, + "34": 1769.0, + "35": 1915.0, + "36": 1885.0, + "37": 2324.0, + "38": 2169.0, + "39": 2300.0, + "40": 2069.0, + "41": 2353.0, + "42": 2236.0, + "43": 1978.0, + "44": 2022.0, + "45": 2103.0, + "46": 2292.0, + "47": 2413.0, + "48": 2305.0, + "49": 2218.0, + "50": 2321.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 522452480.0, + "2": 522452480.0, + "3": 522452480.0, + "4": 522452480.0, + "5": 522452480.0, + "6": 522452480.0, + "7": 522452480.0, + "8": 522452480.0, + "9": 523501056.0, + "10": 522452480.0, + "11": 522452480.0, + "12": 523501056.0, + "13": 522452480.0, + "14": 522452480.0, + "15": 522452480.0, + "16": 522452480.0, + "17": 522452480.0, + "18": 522452480.0, + "19": 523501056.0, + "20": 523501056.0, + "21": 522452480.0, + "22": 522452480.0, + "23": 522452480.0, + "24": 523501056.0, + "25": 522452480.0, + "26": 522452480.0, + "27": 522452480.0, + "28": 522452480.0, + "29": 523501056.0, + "30": 522452480.0, + "31": 522452480.0, + "32": 522452480.0, + "33": 522452480.0, + "34": 522452480.0, + "35": 522452480.0, + "36": 522452480.0, + "37": 522452480.0, + "38": 522452480.0, + "39": 522452480.0, + "40": 522452480.0, + "41": 523371008.0, + "42": 522452480.0, + "43": 522452480.0, + "44": 522452480.0, + "45": 522452480.0, + "46": 523501056.0, + "47": 522452480.0, + "48": 522452480.0, + "49": 523501056.0, + "50": 522452480.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3768846848.0, + "2": 3913263616.0, + "3": 3913263616.0, + "4": 3913263616.0, + "5": 3913263616.0, + "6": 3913263616.0, + "7": 3913263616.0, + "8": 3913263616.0, + "9": 3913263616.0, + "10": 3913263616.0, + "11": 3913263616.0, + "12": 3913263616.0, + "13": 3913263616.0, + "14": 3913263616.0, + "15": 3913263616.0, + "16": 3913263616.0, + "17": 3913263616.0, + "18": 3913263616.0, + "19": 3913263616.0, + "20": 3913263616.0, + "21": 3913263616.0, + "22": 3913263616.0, + "23": 3913263616.0, + "24": 3913263616.0, + "25": 3913263616.0, + "26": 3913263616.0, + "27": 3913263616.0, + "28": 3913263616.0, + "29": 3913263616.0, + "30": 3913263616.0, + "31": 3913263616.0, + "32": 3913263616.0, + "33": 3913263616.0, + "34": 3913263616.0, + "35": 3913263616.0, + "36": 3913263616.0, + "37": 3913263616.0, + "38": 3913263616.0, + "39": 3913263616.0, + "40": 3913263616.0, + "41": 3913263616.0, + "42": 3913263616.0, + "43": 3913263616.0, + "44": 3913263616.0, + "45": 3913263616.0, + "46": 3913263616.0, + "47": 3913263616.0, + "48": 3913263616.0, + "49": 3913263616.0, + "50": 3913263616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.5204, + "2": 0.1877, + "3": 0.15851, + "4": 0.15284, + "5": 0.15092, + "6": 0.15084, + "7": 0.14786, + "8": 0.14787, + "9": 0.14698, + "10": 0.15104, + "11": 0.29695, + "12": 0.14795, + "13": 0.14771, + "14": 0.14662, + "15": 0.14409, + "16": 0.14378, + "17": 0.14431, + "18": 0.14329, + "19": 0.14334, + "20": 0.14441, + "21": 0.28011, + "22": 0.14378, + "23": 0.14643, + "24": 0.14572, + "25": 0.14331, + "26": 0.14307, + "27": 0.14541, + "28": 0.14512, + "29": 0.14536, + "30": 0.14358, + "31": 0.28944, + "32": 0.14533, + "33": 0.14477, + "34": 0.14423, + "35": 0.14395, + "36": 0.14486, + "37": 0.14319, + "38": 0.14455, + "39": 0.14454, + "40": 0.14537, + "41": 0.29312, + "42": 0.14458, + "43": 0.14749, + "44": 0.14448, + "45": 0.14501, + "46": 0.14588, + "47": 0.14249, + "48": 0.14564, + "49": 0.14388, + "50": 0.14222 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json new file mode 100644 index 00000000000..59dee721816 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81271, + "4": 10.78877, + "5": 10.85169, + "6": 10.8704, + "7": 10.831, + "8": 10.83427, + "9": 10.83995, + "10": 10.78684, + "11": 10.88021, + "12": 10.85971, + "13": 10.86589, + "14": 10.87818, + "15": 10.79463, + "16": 10.79607, + "17": 10.7688, + "18": 10.81045, + "19": 10.79836, + "20": 10.69045, + "21": 10.67932, + "22": 10.52101, + "23": 10.70743, + "24": 10.57665, + "25": 10.52275, + "26": 10.595, + "27": 10.5855, + "28": 10.56131, + "29": 10.56894, + "30": 10.34527, + "31": 10.10019, + "32": 10.45229, + "33": 10.44356, + "34": 10.20397, + "35": 10.25844, + "36": 10.2103, + "37": 10.32252, + "38": 10.1661, + "39": 10.38156, + "40": 10.07025, + "41": 10.13542, + "42": 10.19416, + "43": 9.80626, + "44": 9.92627, + "45": 9.8024, + "46": 9.79983, + "47": 10.11662, + "48": 9.81307, + "49": 9.50044, + "50": 9.87631, + "51": 9.82781, + "52": 9.71723, + "53": 10.03979, + "54": 9.92177, + "55": 9.85515, + "56": 9.59253, + "57": 9.44144, + "58": 9.79602, + "59": 9.55567, + "60": 9.4664, + "61": 9.6666, + "62": 9.95363, + "63": 9.33626, + "64": 9.74152, + "65": 8.9178, + "66": 9.66632, + "67": 9.34424, + "68": 9.75273, + "69": 9.75727, + "70": 9.69242, + "71": 9.5868, + "72": 9.55099, + "73": 9.46289, + "74": 8.90671, + "75": 9.37793, + "76": 9.04952, + "77": 10.0301, + "78": 9.69192, + "79": 9.33464, + "80": 9.3667, + "81": 9.44418, + "82": 9.66164, + "83": 9.27209, + "84": 9.38066, + "85": 9.57618, + "86": 9.0424, + "87": 9.55703, + "88": 9.70385, + "89": 9.56619, + "90": 9.77295, + "91": 9.29396, + "92": 9.31912, + "93": 9.03406, + "94": 8.78526, + "95": 9.46938, + "96": 9.47497, + "97": 9.25688, + "98": 9.61835, + "99": 8.83233, + "100": 9.34557 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1668.0, + "4": 1721.0, + "5": 1897.0, + "6": 1823.0, + "7": 1719.0, + "8": 1637.0, + "9": 1742.0, + "10": 1358.0, + "11": 1882.0, + "12": 1781.0, + "13": 1847.0, + "14": 1753.0, + "15": 1883.0, + "16": 1755.0, + "17": 1752.0, + "18": 1683.0, + "19": 1817.0, + "20": 1663.0, + "21": 1795.0, + "22": 1698.0, + "23": 1996.0, + "24": 1620.0, + "25": 1658.0, + "26": 1727.0, + "27": 1781.0, + "28": 2085.0, + "29": 1952.0, + "30": 1821.0, + "31": 1646.0, + "32": 1879.0, + "33": 2034.0, + "34": 1861.0, + "35": 1834.0, + "36": 1913.0, + "37": 2333.0, + "38": 2070.0, + "39": 2245.0, + "40": 2126.0, + "41": 2311.0, + "42": 2213.0, + "43": 1907.0, + "44": 1951.0, + "45": 2001.0, + "46": 2218.0, + "47": 2533.0, + "48": 2436.0, + "49": 2188.0, + "50": 2342.0, + "51": 2562.0, + "52": 2529.0, + "53": 3031.0, + "54": 2744.0, + "55": 2264.0, + "56": 2794.0, + "57": 2183.0, + "58": 2882.0, + "59": 2769.0, + "60": 2399.0, + "61": 3031.0, + "62": 2706.0, + "63": 2388.0, + "64": 3046.0, + "65": 2597.0, + "66": 3092.0, + "67": 2730.0, + "68": 2858.0, + "69": 2982.0, + "70": 3202.0, + "71": 2964.0, + "72": 2450.0, + "73": 2817.0, + "74": 1834.0, + "75": 2609.0, + "76": 3000.0, + "77": 3180.0, + "78": 3113.0, + "79": 3145.0, + "80": 3258.0, + "81": 3645.0, + "82": 3075.0, + "83": 2812.0, + "84": 3295.0, + "85": 3368.0, + "86": 2730.0, + "87": 3717.0, + "88": 3056.0, + "89": 3252.0, + "90": 2954.0, + "91": 2798.0, + "92": 3089.0, + "93": 2742.0, + "94": 3420.0, + "95": 3225.0, + "96": 3362.0, + "97": 3118.0, + "98": 3671.0, + "99": 3341.0, + "100": 3428.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2366437376.0, + "2": 2648148992.0, + "3": 2648148992.0, + "4": 2648148992.0, + "5": 2648148992.0, + "6": 2648148992.0, + "7": 2648148992.0, + "8": 2648148992.0, + "9": 2648148992.0, + "10": 2648148992.0, + "11": 2648148992.0, + "12": 2648148992.0, + "13": 2648148992.0, + "14": 2648148992.0, + "15": 2648148992.0, + "16": 2648148992.0, + "17": 2648148992.0, + "18": 2648148992.0, + "19": 2648148992.0, + "20": 2648148992.0, + "21": 2648148992.0, + "22": 2648148992.0, + "23": 2648148992.0, + "24": 2648148992.0, + "25": 2648148992.0, + "26": 2648148992.0, + "27": 2648148992.0, + "28": 2648148992.0, + "29": 2648148992.0, + "30": 2648148992.0, + "31": 2648148992.0, + "32": 2648148992.0, + "33": 2648148992.0, + "34": 2648148992.0, + "35": 2648148992.0, + "36": 2648148992.0, + "37": 2648148992.0, + "38": 2648148992.0, + "39": 2648148992.0, + "40": 2648148992.0, + "41": 2648148992.0, + "42": 2648148992.0, + "43": 2648148992.0, + "44": 2648148992.0, + "45": 2648148992.0, + "46": 2648148992.0, + "47": 2648148992.0, + "48": 2648148992.0, + "49": 2648148992.0, + "50": 2648148992.0, + "51": 2648148992.0, + "52": 2648148992.0, + "53": 2648148992.0, + "54": 2648148992.0, + "55": 2648148992.0, + "56": 2648148992.0, + "57": 2648148992.0, + "58": 2648148992.0, + "59": 2648148992.0, + "60": 2648148992.0, + "61": 2648148992.0, + "62": 2648148992.0, + "63": 2648148992.0, + "64": 2648148992.0, + "65": 2648148992.0, + "66": 2648148992.0, + "67": 2648148992.0, + "68": 2648148992.0, + "69": 2648148992.0, + "70": 2648148992.0, + "71": 2648148992.0, + "72": 2648148992.0, + "73": 2648148992.0, + "74": 2648148992.0, + "75": 2648148992.0, + "76": 2648148992.0, + "77": 2648148992.0, + "78": 2648148992.0, + "79": 2648148992.0, + "80": 2648148992.0, + "81": 2648148992.0, + "82": 2648148992.0, + "83": 2648148992.0, + "84": 2648148992.0, + "85": 2648148992.0, + "86": 2648148992.0, + "87": 2648148992.0, + "88": 2648148992.0, + "89": 2648148992.0, + "90": 2648148992.0, + "91": 2648148992.0, + "92": 2648148992.0, + "93": 2648148992.0, + "94": 2648148992.0, + "95": 2648148992.0, + "96": 2648148992.0, + "97": 2648148992.0, + "98": 2648148992.0, + "99": 2648148992.0, + "100": 2648148992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.77273, + "2": 0.15704, + "3": 0.19484, + "4": 0.13176, + "5": 0.13019, + "6": 0.12976, + "7": 0.1302, + "8": 0.12925, + "9": 0.12988, + "10": 0.13099, + "11": 0.13015, + "12": 0.1297, + "13": 0.12988, + "14": 0.13024, + "15": 0.12985, + "16": 0.12971, + "17": 0.12961, + "18": 0.1302, + "19": 0.12963, + "20": 0.12994, + "21": 0.1299, + "22": 0.13037, + "23": 0.13043, + "24": 0.12989, + "25": 0.13018, + "26": 0.13019, + "27": 0.12985, + "28": 0.13014, + "29": 0.13068, + "30": 0.13099, + "31": 0.13197, + "32": 0.13151, + "33": 0.13168, + "34": 0.1303, + "35": 0.13073, + "36": 0.13088, + "37": 0.1307, + "38": 0.13091, + "39": 0.13292, + "40": 0.13172, + "41": 0.134, + "42": 0.13157, + "43": 0.13272, + "44": 0.13144, + "45": 0.13142, + "46": 0.133, + "47": 0.13069, + "48": 0.13192, + "49": 0.13124, + "50": 0.13106, + "51": 0.13227, + "52": 0.13218, + "53": 0.13063, + "54": 0.13182, + "55": 0.13138, + "56": 0.13226, + "57": 0.13156, + "58": 0.13127, + "59": 0.13198, + "60": 0.13133, + "61": 0.13107, + "62": 0.13121, + "63": 0.13141, + "64": 0.13149, + "65": 0.13192, + "66": 0.13188, + "67": 0.13167, + "68": 0.1319, + "69": 0.1318, + "70": 0.13204, + "71": 0.13077, + "72": 0.13205, + "73": 0.13251, + "74": 0.13212, + "75": 0.13195, + "76": 0.13016, + "77": 0.12994, + "78": 0.13184, + "79": 0.13131, + "80": 0.13153, + "81": 0.13219, + "82": 0.13088, + "83": 0.13074, + "84": 0.13035, + "85": 0.12939, + "86": 0.12953, + "87": 0.1294, + "88": 0.12951, + "89": 0.13007, + "90": 0.12893, + "91": 0.13881, + "92": 0.1299, + "93": 0.12993, + "94": 0.13022, + "95": 0.1304, + "96": 0.12965, + "97": 0.13013, + "98": 0.1306, + "99": 0.12958, + "100": 0.13011 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json index 855eefe74c4..6081b627567 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85285, "10": 10.78449, "15": 10.79226, "20": 10.69196, "25": 10.52318, "30": 10.34505, "35": 10.25888, "40": 10.07032, "45": 9.803, "50": 9.87675, "55": 9.85523, "60": 9.46634, "65": 8.91657, "70": 9.69276, "75": 9.37813, "80": 9.368, "85": 9.57598, "90": 9.77247, "95": 9.46912, "100": 9.34574}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1915.0, "10": 1361.0, "15": 1831.0, "20": 1695.0, "25": 1650.0, "30": 1861.0, "35": 1916.0, "40": 2109.0, "45": 2098.0, "50": 2424.0, "55": 2309.0, "60": 2432.0, "65": 2660.0, "70": 3132.0, "75": 2555.0, "80": 3370.0, "85": 3434.0, "90": 2868.0, "95": 3273.0, "100": 3359.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0, "55": 732811264.0, "60": 732811264.0, "65": 732811264.0, "70": 732811264.0, "75": 732811264.0, "80": 732811264.0, "85": 732811264.0, "90": 732811264.0, "95": 732811264.0, "100": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0, "55": 4122703872.0, "60": 4122703872.0, "65": 4122703872.0, "70": 4122703872.0, "75": 4122703872.0, "80": 4122703872.0, "85": 4122703872.0, "90": 4122703872.0, "95": 4122703872.0, "100": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 21.12849, "5": 0.1796, "10": 0.17779, "15": 0.17478, "20": 0.17462, "25": 0.17506, "30": 0.17537, "35": 0.17332, "40": 0.17629, "45": 0.17732, "50": 0.17437, "55": 0.1738, "60": 0.17515, "65": 0.17451, "70": 0.17318, "75": 0.1729, "80": 0.17444, "85": 0.17406, "90": 0.17228, "95": 0.17346, "100": 0.17333}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81271, + "4": 10.78877, + "5": 10.85169, + "6": 10.8704, + "7": 10.831, + "8": 10.83427, + "9": 10.83995, + "10": 10.78684, + "11": 10.88021, + "12": 10.85971, + "13": 10.86589, + "14": 10.87818, + "15": 10.79463, + "16": 10.79607, + "17": 10.7688, + "18": 10.81045, + "19": 10.79836, + "20": 10.69045, + "21": 10.67932, + "22": 10.52101, + "23": 10.70743, + "24": 10.57665, + "25": 10.52275, + "26": 10.595, + "27": 10.5855, + "28": 10.56131, + "29": 10.56894, + "30": 10.34527, + "31": 10.10019, + "32": 10.45229, + "33": 10.44356, + "34": 10.20397, + "35": 10.25844, + "36": 10.2103, + "37": 10.32252, + "38": 10.1661, + "39": 10.38156, + "40": 10.07025, + "41": 10.13542, + "42": 10.19416, + "43": 9.80626, + "44": 9.92627, + "45": 9.8024, + "46": 9.79983, + "47": 10.11662, + "48": 9.81307, + "49": 9.50044, + "50": 9.87631, + "51": 9.82781, + "52": 9.71723, + "53": 10.03979, + "54": 9.92177, + "55": 9.85515, + "56": 9.59253, + "57": 9.44144, + "58": 9.79602, + "59": 9.55567, + "60": 9.4664, + "61": 9.6666, + "62": 9.95363, + "63": 9.33626, + "64": 9.74152, + "65": 8.9178, + "66": 9.66632, + "67": 9.34424, + "68": 9.75273, + "69": 9.75727, + "70": 9.69242, + "71": 9.5868, + "72": 9.55099, + "73": 9.46289, + "74": 8.90671, + "75": 9.37793, + "76": 9.04952, + "77": 10.0301, + "78": 9.69192, + "79": 9.33464, + "80": 9.3667, + "81": 9.44418, + "82": 9.66164, + "83": 9.27209, + "84": 9.38066, + "85": 9.57618, + "86": 9.0424, + "87": 9.55703, + "88": 9.70385, + "89": 9.56619, + "90": 9.77295, + "91": 9.29396, + "92": 9.31912, + "93": 9.03406, + "94": 8.78526, + "95": 9.46938, + "96": 9.47497, + "97": 9.25688, + "98": 9.61835, + "99": 8.83233, + "100": 9.34557 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1668.0, + "4": 1721.0, + "5": 1897.0, + "6": 1823.0, + "7": 1719.0, + "8": 1637.0, + "9": 1742.0, + "10": 1358.0, + "11": 1882.0, + "12": 1781.0, + "13": 1847.0, + "14": 1753.0, + "15": 1883.0, + "16": 1755.0, + "17": 1752.0, + "18": 1683.0, + "19": 1817.0, + "20": 1663.0, + "21": 1795.0, + "22": 1698.0, + "23": 1996.0, + "24": 1620.0, + "25": 1658.0, + "26": 1727.0, + "27": 1781.0, + "28": 2085.0, + "29": 1952.0, + "30": 1821.0, + "31": 1646.0, + "32": 1879.0, + "33": 2034.0, + "34": 1861.0, + "35": 1834.0, + "36": 1913.0, + "37": 2333.0, + "38": 2070.0, + "39": 2245.0, + "40": 2126.0, + "41": 2311.0, + "42": 2213.0, + "43": 1907.0, + "44": 1951.0, + "45": 2001.0, + "46": 2218.0, + "47": 2533.0, + "48": 2436.0, + "49": 2188.0, + "50": 2342.0, + "51": 2562.0, + "52": 2529.0, + "53": 3031.0, + "54": 2744.0, + "55": 2264.0, + "56": 2794.0, + "57": 2183.0, + "58": 2882.0, + "59": 2769.0, + "60": 2399.0, + "61": 3031.0, + "62": 2706.0, + "63": 2388.0, + "64": 3046.0, + "65": 2597.0, + "66": 3092.0, + "67": 2730.0, + "68": 2858.0, + "69": 2982.0, + "70": 3202.0, + "71": 2964.0, + "72": 2450.0, + "73": 2817.0, + "74": 1834.0, + "75": 2609.0, + "76": 3000.0, + "77": 3180.0, + "78": 3113.0, + "79": 3145.0, + "80": 3258.0, + "81": 3645.0, + "82": 3075.0, + "83": 2812.0, + "84": 3295.0, + "85": 3368.0, + "86": 2730.0, + "87": 3717.0, + "88": 3056.0, + "89": 3252.0, + "90": 2954.0, + "91": 2798.0, + "92": 3089.0, + "93": 2742.0, + "94": 3420.0, + "95": 3225.0, + "96": 3362.0, + "97": 3118.0, + "98": 3671.0, + "99": 3341.0, + "100": 3428.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2366437376.0, + "2": 2648148992.0, + "3": 2648148992.0, + "4": 2648148992.0, + "5": 2648148992.0, + "6": 2648148992.0, + "7": 2648148992.0, + "8": 2648148992.0, + "9": 2648148992.0, + "10": 2648148992.0, + "11": 2648148992.0, + "12": 2648148992.0, + "13": 2648148992.0, + "14": 2648148992.0, + "15": 2648148992.0, + "16": 2648148992.0, + "17": 2648148992.0, + "18": 2648148992.0, + "19": 2648148992.0, + "20": 2648148992.0, + "21": 2648148992.0, + "22": 2648148992.0, + "23": 2648148992.0, + "24": 2648148992.0, + "25": 2648148992.0, + "26": 2648148992.0, + "27": 2648148992.0, + "28": 2648148992.0, + "29": 2648148992.0, + "30": 2648148992.0, + "31": 2648148992.0, + "32": 2648148992.0, + "33": 2648148992.0, + "34": 2648148992.0, + "35": 2648148992.0, + "36": 2648148992.0, + "37": 2648148992.0, + "38": 2648148992.0, + "39": 2648148992.0, + "40": 2648148992.0, + "41": 2648148992.0, + "42": 2648148992.0, + "43": 2648148992.0, + "44": 2648148992.0, + "45": 2648148992.0, + "46": 2648148992.0, + "47": 2648148992.0, + "48": 2648148992.0, + "49": 2648148992.0, + "50": 2648148992.0, + "51": 2648148992.0, + "52": 2648148992.0, + "53": 2648148992.0, + "54": 2648148992.0, + "55": 2648148992.0, + "56": 2648148992.0, + "57": 2648148992.0, + "58": 2648148992.0, + "59": 2648148992.0, + "60": 2648148992.0, + "61": 2648148992.0, + "62": 2648148992.0, + "63": 2648148992.0, + "64": 2648148992.0, + "65": 2648148992.0, + "66": 2648148992.0, + "67": 2648148992.0, + "68": 2648148992.0, + "69": 2648148992.0, + "70": 2648148992.0, + "71": 2648148992.0, + "72": 2648148992.0, + "73": 2648148992.0, + "74": 2648148992.0, + "75": 2648148992.0, + "76": 2648148992.0, + "77": 2648148992.0, + "78": 2648148992.0, + "79": 2648148992.0, + "80": 2648148992.0, + "81": 2648148992.0, + "82": 2648148992.0, + "83": 2648148992.0, + "84": 2648148992.0, + "85": 2648148992.0, + "86": 2648148992.0, + "87": 2648148992.0, + "88": 2648148992.0, + "89": 2648148992.0, + "90": 2648148992.0, + "91": 2648148992.0, + "92": 2648148992.0, + "93": 2648148992.0, + "94": 2648148992.0, + "95": 2648148992.0, + "96": 2648148992.0, + "97": 2648148992.0, + "98": 2648148992.0, + "99": 2648148992.0, + "100": 2648148992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.81482, + "2": 0.16445, + "3": 0.16681, + "4": 0.12923, + "5": 0.12855, + "6": 0.12774, + "7": 0.12794, + "8": 0.12857, + "9": 0.12785, + "10": 0.12889, + "11": 0.13344, + "12": 0.1302, + "13": 0.13007, + "14": 0.12962, + "15": 0.13044, + "16": 0.12918, + "17": 0.13075, + "18": 0.13004, + "19": 0.13052, + "20": 0.13025, + "21": 0.12825, + "22": 0.13322, + "23": 0.13274, + "24": 0.13114, + "25": 0.13075, + "26": 0.12979, + "27": 0.13026, + "28": 0.13147, + "29": 0.13072, + "30": 0.13098, + "31": 0.13095, + "32": 0.13054, + "33": 0.13038, + "34": 0.13142, + "35": 0.13065, + "36": 0.12923, + "37": 0.13039, + "38": 0.12981, + "39": 0.12995, + "40": 0.13035, + "41": 0.12966, + "42": 0.13013, + "43": 0.13031, + "44": 0.13066, + "45": 0.12952, + "46": 0.13059, + "47": 0.12932, + "48": 0.13133, + "49": 0.13099, + "50": 0.13032, + "51": 0.13345, + "52": 0.13027, + "53": 0.13035, + "54": 0.13064, + "55": 0.13026, + "56": 0.13053, + "57": 0.13106, + "58": 0.13032, + "59": 0.13178, + "60": 0.13233, + "61": 0.13005, + "62": 0.13045, + "63": 0.13061, + "64": 0.13066, + "65": 0.13102, + "66": 0.13143, + "67": 0.13033, + "68": 0.13066, + "69": 0.12904, + "70": 0.13059, + "71": 0.13052, + "72": 0.13076, + "73": 0.13215, + "74": 0.13173, + "75": 0.13126, + "76": 0.12946, + "77": 0.13071, + "78": 0.12973, + "79": 0.12962, + "80": 0.12976, + "81": 0.12993, + "82": 0.12829, + "83": 0.13132, + "84": 0.1304, + "85": 0.13095, + "86": 0.13112, + "87": 0.12994, + "88": 0.13287, + "89": 0.1284, + "90": 0.1303, + "91": 0.12966, + "92": 0.13139, + "93": 0.12932, + "94": 0.12687, + "95": 0.13012, + "96": 0.12919, + "97": 0.13166, + "98": 0.12958, + "99": 0.13126, + "100": 0.1303 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json new file mode 100644 index 00000000000..f108f63200b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79613, + "17": 10.76887, + "18": 10.81076, + "19": 10.7986, + "20": 10.69061, + "21": 10.67945, + "22": 10.52114, + "23": 10.70755, + "24": 10.57741, + "25": 10.52373, + "26": 10.59597, + "27": 10.58619, + "28": 10.56176, + "29": 10.56945, + "30": 10.34573, + "31": 10.10078, + "32": 10.45392, + "33": 10.4461, + "34": 10.20721, + "35": 10.262, + "36": 10.21312, + "37": 10.32468, + "38": 10.16831, + "39": 10.38374, + "40": 10.07231, + "41": 10.13763, + "42": 10.19765, + "43": 9.81155, + "44": 9.93311, + "45": 9.8104, + "46": 9.80854, + "47": 10.12558, + "48": 9.82105, + "49": 9.50764, + "50": 9.88382, + "51": 9.83549, + "52": 9.72516, + "53": 10.04799, + "54": 9.93011, + "55": 9.8636, + "56": 9.60217, + "57": 9.45187, + "58": 9.8078, + "59": 9.56783, + "60": 9.47966, + "61": 9.67984, + "62": 9.96754, + "63": 9.35113, + "64": 9.75623, + "65": 8.9318, + "66": 9.68107, + "67": 9.35956, + "68": 9.76948, + "69": 9.77492, + "70": 9.71182, + "71": 9.60632, + "72": 9.57129, + "73": 9.48392, + "74": 8.92911, + "75": 9.40028, + "76": 9.07194, + "77": 10.05252, + "78": 9.71494, + "79": 9.35747, + "80": 9.38946, + "81": 9.46791, + "82": 9.68508, + "83": 9.29588, + "84": 9.40522, + "85": 9.60163, + "86": 9.06713, + "87": 9.58402, + "88": 9.73304, + "89": 9.59526, + "90": 9.80555, + "91": 9.32604, + "92": 9.35323, + "93": 9.06915, + "94": 8.82268, + "95": 9.50858, + "96": 9.51584, + "97": 9.2976, + "98": 9.66184, + "99": 8.87662, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1728.0, + "16": 1773.0, + "17": 1805.0, + "18": 1702.0, + "19": 1803.0, + "20": 1627.0, + "21": 1822.0, + "22": 1748.0, + "23": 1938.0, + "24": 1642.0, + "25": 1696.0, + "26": 1760.0, + "27": 1809.0, + "28": 2025.0, + "29": 1900.0, + "30": 1902.0, + "31": 1645.0, + "32": 1876.0, + "33": 2105.0, + "34": 1881.0, + "35": 1913.0, + "36": 1864.0, + "37": 2322.0, + "38": 2194.0, + "39": 2318.0, + "40": 2010.0, + "41": 2358.0, + "42": 2155.0, + "43": 1980.0, + "44": 2105.0, + "45": 2082.0, + "46": 2221.0, + "47": 2537.0, + "48": 2367.0, + "49": 2190.0, + "50": 2352.0, + "51": 2441.0, + "52": 2482.0, + "53": 2916.0, + "54": 2550.0, + "55": 2347.0, + "56": 2765.0, + "57": 2116.0, + "58": 2968.0, + "59": 2810.0, + "60": 2384.0, + "61": 2912.0, + "62": 2554.0, + "63": 2364.0, + "64": 3035.0, + "65": 2648.0, + "66": 2979.0, + "67": 2741.0, + "68": 2799.0, + "69": 3071.0, + "70": 3098.0, + "71": 2950.0, + "72": 2342.0, + "73": 2829.0, + "74": 1840.0, + "75": 2426.0, + "76": 2941.0, + "77": 3245.0, + "78": 3272.0, + "79": 3066.0, + "80": 3221.0, + "81": 3565.0, + "82": 3162.0, + "83": 2876.0, + "84": 3180.0, + "85": 3410.0, + "86": 2778.0, + "87": 3752.0, + "88": 2995.0, + "89": 3264.0, + "90": 2940.0, + "91": 2791.0, + "92": 3118.0, + "93": 2634.0, + "94": 3464.0, + "95": 3344.0, + "96": 3499.0, + "97": 3122.0, + "98": 3568.0, + "99": 3272.0, + "100": 3476.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 733860352.0, + "2": 733860352.0, + "3": 733860352.0, + "4": 733860352.0, + "5": 733860352.0, + "6": 733860352.0, + "7": 733860352.0, + "8": 733860352.0, + "9": 733860352.0, + "10": 733860352.0, + "11": 733860352.0, + "12": 733860352.0, + "13": 733860352.0, + "14": 733860352.0, + "15": 733860352.0, + "16": 733860352.0, + "17": 733860352.0, + "18": 733860352.0, + "19": 733860352.0, + "20": 733860352.0, + "21": 733860352.0, + "22": 733860352.0, + "23": 733860352.0, + "24": 733860352.0, + "25": 733860352.0, + "26": 733860352.0, + "27": 733860352.0, + "28": 733860352.0, + "29": 733860352.0, + "30": 733860352.0, + "31": 733860352.0, + "32": 733860352.0, + "33": 733860352.0, + "34": 733860352.0, + "35": 733860352.0, + "36": 733860352.0, + "37": 733860352.0, + "38": 733860352.0, + "39": 733860352.0, + "40": 733860352.0, + "41": 733860352.0, + "42": 733860352.0, + "43": 733860352.0, + "44": 733860352.0, + "45": 733860352.0, + "46": 733860352.0, + "47": 733860352.0, + "48": 733860352.0, + "49": 733860352.0, + "50": 733860352.0, + "51": 733860352.0, + "52": 733860352.0, + "53": 733860352.0, + "54": 733860352.0, + "55": 733860352.0, + "56": 733860352.0, + "57": 733860352.0, + "58": 733860352.0, + "59": 733860352.0, + "60": 733860352.0, + "61": 733860352.0, + "62": 733860352.0, + "63": 733860352.0, + "64": 733860352.0, + "65": 733860352.0, + "66": 733860352.0, + "67": 733860352.0, + "68": 733860352.0, + "69": 733860352.0, + "70": 733860352.0, + "71": 733860352.0, + "72": 733860352.0, + "73": 733860352.0, + "74": 733860352.0, + "75": 733860352.0, + "76": 733860352.0, + "77": 733860352.0, + "78": 733860352.0, + "79": 733860352.0, + "80": 733860352.0, + "81": 733860352.0, + "82": 733860352.0, + "83": 733860352.0, + "84": 733860352.0, + "85": 733860352.0, + "86": 733860352.0, + "87": 733860352.0, + "88": 733860352.0, + "89": 733860352.0, + "90": 733860352.0, + "91": 733860352.0, + "92": 733860352.0, + "93": 733860352.0, + "94": 733860352.0, + "95": 733860352.0, + "96": 733860352.0, + "97": 733860352.0, + "98": 733860352.0, + "99": 733860352.0, + "100": 733860352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2367879168.0, + "2": 2651687936.0, + "3": 2651687936.0, + "4": 2651687936.0, + "5": 2651687936.0, + "6": 2651687936.0, + "7": 2651687936.0, + "8": 2651687936.0, + "9": 2651687936.0, + "10": 2651687936.0, + "11": 2651687936.0, + "12": 2651687936.0, + "13": 2651687936.0, + "14": 2651687936.0, + "15": 2651687936.0, + "16": 2651687936.0, + "17": 2651687936.0, + "18": 2651687936.0, + "19": 2651687936.0, + "20": 2651687936.0, + "21": 2651687936.0, + "22": 2651687936.0, + "23": 2651687936.0, + "24": 2651687936.0, + "25": 2651687936.0, + "26": 2651687936.0, + "27": 2651687936.0, + "28": 2651687936.0, + "29": 2651687936.0, + "30": 2651687936.0, + "31": 2651687936.0, + "32": 2651687936.0, + "33": 2651687936.0, + "34": 2651687936.0, + "35": 2651687936.0, + "36": 2651687936.0, + "37": 2651687936.0, + "38": 2651687936.0, + "39": 2651687936.0, + "40": 2651687936.0, + "41": 2651687936.0, + "42": 2651687936.0, + "43": 2651687936.0, + "44": 2651687936.0, + "45": 2651687936.0, + "46": 2651687936.0, + "47": 2651687936.0, + "48": 2651687936.0, + "49": 2651687936.0, + "50": 2651687936.0, + "51": 2651687936.0, + "52": 2651687936.0, + "53": 2651687936.0, + "54": 2651687936.0, + "55": 2651687936.0, + "56": 2651687936.0, + "57": 2651687936.0, + "58": 2651687936.0, + "59": 2651687936.0, + "60": 2651687936.0, + "61": 2651687936.0, + "62": 2651687936.0, + "63": 2651687936.0, + "64": 2651687936.0, + "65": 2651687936.0, + "66": 2651687936.0, + "67": 2651687936.0, + "68": 2651687936.0, + "69": 2651687936.0, + "70": 2651687936.0, + "71": 2651687936.0, + "72": 2651687936.0, + "73": 2651687936.0, + "74": 2651687936.0, + "75": 2651687936.0, + "76": 2651687936.0, + "77": 2651687936.0, + "78": 2651687936.0, + "79": 2651687936.0, + "80": 2651687936.0, + "81": 2651687936.0, + "82": 2651687936.0, + "83": 2651687936.0, + "84": 2651687936.0, + "85": 2651687936.0, + "86": 2651687936.0, + "87": 2651687936.0, + "88": 2651687936.0, + "89": 2651687936.0, + "90": 2651687936.0, + "91": 2651687936.0, + "92": 2651687936.0, + "93": 2651687936.0, + "94": 2651687936.0, + "95": 2651687936.0, + "96": 2651687936.0, + "97": 2651687936.0, + "98": 2651687936.0, + "99": 2651687936.0, + "100": 2651687936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.1008, + "2": 0.16051, + "3": 0.12978, + "4": 0.12855, + "5": 0.12836, + "6": 0.12718, + "7": 0.12817, + "8": 0.12827, + "9": 0.12773, + "10": 0.12934, + "11": 0.1284, + "12": 0.1278, + "13": 0.12824, + "14": 0.12897, + "15": 0.12788, + "16": 0.12662, + "17": 0.12751, + "18": 0.12678, + "19": 0.12784, + "20": 0.12756, + "21": 0.12782, + "22": 0.12765, + "23": 0.12695, + "24": 0.12621, + "25": 0.12639, + "26": 0.12652, + "27": 0.1261, + "28": 0.12599, + "29": 0.12679, + "30": 0.12648, + "31": 0.12791, + "32": 0.1267, + "33": 0.12736, + "34": 0.1275, + "35": 0.12674, + "36": 0.12623, + "37": 0.12561, + "38": 0.12629, + "39": 0.12735, + "40": 0.12739, + "41": 0.12784, + "42": 0.12763, + "43": 0.12841, + "44": 0.12666, + "45": 0.12797, + "46": 0.12722, + "47": 0.12583, + "48": 0.1271, + "49": 0.12675, + "50": 0.12769, + "51": 0.13852, + "52": 0.1338, + "53": 0.1334, + "54": 0.13466, + "55": 0.13471, + "56": 0.13266, + "57": 0.13116, + "58": 0.13405, + "59": 0.13356, + "60": 0.13304, + "61": 0.13328, + "62": 0.13243, + "63": 0.13255, + "64": 0.13344, + "65": 0.13433, + "66": 0.13486, + "67": 0.13338, + "68": 0.13313, + "69": 0.13327, + "70": 0.1324, + "71": 0.13325, + "72": 0.13418, + "73": 0.1341, + "74": 0.1334, + "75": 0.13238, + "76": 0.13198, + "77": 0.13412, + "78": 0.1335, + "79": 0.13208, + "80": 0.13334, + "81": 0.13338, + "82": 0.13187, + "83": 0.13324, + "84": 0.13268, + "85": 0.13362, + "86": 0.13282, + "87": 0.13325, + "88": 0.13348, + "89": 0.13361, + "90": 0.13267, + "91": 0.13322, + "92": 0.13404, + "93": 0.13424, + "94": 0.13249, + "95": 0.1323, + "96": 0.13217, + "97": 0.16026, + "98": 0.13491, + "99": 0.13704, + "100": 0.13716 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json index fbeaf7f4f0d..0cb12854799 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69211, "25": 10.52412, "30": 10.34552, "35": 10.26239, "40": 10.07241, "45": 9.81101, "50": 9.88422, "55": 9.86374, "60": 9.47965, "65": 8.93063, "70": 9.71215, "75": 9.40048, "80": 9.39077, "85": 9.60141, "90": 9.80501, "95": 9.50816, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1601.0, "25": 1635.0, "30": 1936.0, "35": 1908.0, "40": 2100.0, "45": 2098.0, "50": 2333.0, "55": 2260.0, "60": 2399.0, "65": 2656.0, "70": 3077.0, "75": 2547.0, "80": 3315.0, "85": 3371.0, "90": 2943.0, "95": 3457.0, "100": 3292.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0, "55": 732811264.0, "60": 732811264.0, "65": 732811264.0, "70": 732811264.0, "75": 732811264.0, "80": 732811264.0, "85": 732811264.0, "90": 732811264.0, "95": 732811264.0, "100": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0, "55": 4122703872.0, "60": 4122703872.0, "65": 4122703872.0, "70": 4122703872.0, "75": 4122703872.0, "80": 4122703872.0, "85": 4122703872.0, "90": 4122703872.0, "95": 4122703872.0, "100": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 24.14685, "5": 0.1855, "10": 0.18669, "15": 0.18585, "20": 0.18594, "25": 0.18564, "30": 0.18382, "35": 0.18679, "40": 0.1866, "45": 0.18389, "50": 0.18294, "55": 0.17556, "60": 0.17509, "65": 0.17542, "70": 0.17558, "75": 0.176, "80": 0.17523, "85": 0.17384, "90": 0.17335, "95": 0.17222, "100": 0.17356}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79613, + "17": 10.76887, + "18": 10.81076, + "19": 10.7986, + "20": 10.69061, + "21": 10.67945, + "22": 10.52114, + "23": 10.70755, + "24": 10.57741, + "25": 10.52373, + "26": 10.59597, + "27": 10.58619, + "28": 10.56176, + "29": 10.56945, + "30": 10.34573, + "31": 10.10078, + "32": 10.45392, + "33": 10.4461, + "34": 10.20721, + "35": 10.262, + "36": 10.21312, + "37": 10.32468, + "38": 10.16831, + "39": 10.38374, + "40": 10.07231, + "41": 10.13763, + "42": 10.19765, + "43": 9.81155, + "44": 9.93311, + "45": 9.8104, + "46": 9.80854, + "47": 10.12558, + "48": 9.82105, + "49": 9.50764, + "50": 9.88382, + "51": 9.83549, + "52": 9.72516, + "53": 10.04799, + "54": 9.93011, + "55": 9.8636, + "56": 9.60217, + "57": 9.45187, + "58": 9.8078, + "59": 9.56783, + "60": 9.47966, + "61": 9.67984, + "62": 9.96754, + "63": 9.35113, + "64": 9.75623, + "65": 8.9318, + "66": 9.68107, + "67": 9.35956, + "68": 9.76948, + "69": 9.77492, + "70": 9.71182, + "71": 9.60632, + "72": 9.57129, + "73": 9.48392, + "74": 8.92911, + "75": 9.40028, + "76": 9.07194, + "77": 10.05252, + "78": 9.71494, + "79": 9.35747, + "80": 9.38946, + "81": 9.46791, + "82": 9.68508, + "83": 9.29588, + "84": 9.40522, + "85": 9.60163, + "86": 9.06713, + "87": 9.58402, + "88": 9.73304, + "89": 9.59526, + "90": 9.80555, + "91": 9.32604, + "92": 9.35323, + "93": 9.06915, + "94": 8.82268, + "95": 9.50858, + "96": 9.51584, + "97": 9.2976, + "98": 9.66184, + "99": 8.87662, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1728.0, + "16": 1773.0, + "17": 1805.0, + "18": 1702.0, + "19": 1803.0, + "20": 1627.0, + "21": 1822.0, + "22": 1748.0, + "23": 1938.0, + "24": 1642.0, + "25": 1696.0, + "26": 1760.0, + "27": 1809.0, + "28": 2025.0, + "29": 1900.0, + "30": 1902.0, + "31": 1645.0, + "32": 1876.0, + "33": 2105.0, + "34": 1881.0, + "35": 1913.0, + "36": 1864.0, + "37": 2322.0, + "38": 2194.0, + "39": 2318.0, + "40": 2010.0, + "41": 2358.0, + "42": 2155.0, + "43": 1980.0, + "44": 2105.0, + "45": 2082.0, + "46": 2221.0, + "47": 2537.0, + "48": 2367.0, + "49": 2190.0, + "50": 2352.0, + "51": 2441.0, + "52": 2482.0, + "53": 2916.0, + "54": 2550.0, + "55": 2347.0, + "56": 2765.0, + "57": 2116.0, + "58": 2968.0, + "59": 2810.0, + "60": 2384.0, + "61": 2912.0, + "62": 2554.0, + "63": 2364.0, + "64": 3035.0, + "65": 2648.0, + "66": 2979.0, + "67": 2741.0, + "68": 2799.0, + "69": 3071.0, + "70": 3098.0, + "71": 2950.0, + "72": 2342.0, + "73": 2829.0, + "74": 1840.0, + "75": 2426.0, + "76": 2941.0, + "77": 3245.0, + "78": 3272.0, + "79": 3066.0, + "80": 3221.0, + "81": 3565.0, + "82": 3162.0, + "83": 2876.0, + "84": 3180.0, + "85": 3410.0, + "86": 2778.0, + "87": 3752.0, + "88": 2995.0, + "89": 3264.0, + "90": 2940.0, + "91": 2791.0, + "92": 3118.0, + "93": 2634.0, + "94": 3464.0, + "95": 3344.0, + "96": 3499.0, + "97": 3122.0, + "98": 3568.0, + "99": 3272.0, + "100": 3476.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 733860352.0, + "2": 733860352.0, + "3": 733860352.0, + "4": 733860352.0, + "5": 733860352.0, + "6": 733860352.0, + "7": 733860352.0, + "8": 733860352.0, + "9": 733860352.0, + "10": 733860352.0, + "11": 733860352.0, + "12": 733860352.0, + "13": 733860352.0, + "14": 733860352.0, + "15": 733860352.0, + "16": 733860352.0, + "17": 733860352.0, + "18": 733860352.0, + "19": 733860352.0, + "20": 733860352.0, + "21": 733860352.0, + "22": 733860352.0, + "23": 733860352.0, + "24": 733860352.0, + "25": 733860352.0, + "26": 733860352.0, + "27": 733860352.0, + "28": 733860352.0, + "29": 733860352.0, + "30": 733860352.0, + "31": 733860352.0, + "32": 733860352.0, + "33": 733860352.0, + "34": 733860352.0, + "35": 733860352.0, + "36": 733860352.0, + "37": 733860352.0, + "38": 733860352.0, + "39": 733860352.0, + "40": 733860352.0, + "41": 733860352.0, + "42": 733860352.0, + "43": 733860352.0, + "44": 733860352.0, + "45": 733860352.0, + "46": 733860352.0, + "47": 733860352.0, + "48": 733860352.0, + "49": 733860352.0, + "50": 733860352.0, + "51": 733860352.0, + "52": 733860352.0, + "53": 733860352.0, + "54": 733860352.0, + "55": 733860352.0, + "56": 733860352.0, + "57": 733860352.0, + "58": 733860352.0, + "59": 733860352.0, + "60": 733860352.0, + "61": 733860352.0, + "62": 733860352.0, + "63": 733860352.0, + "64": 733860352.0, + "65": 733860352.0, + "66": 733860352.0, + "67": 733860352.0, + "68": 733860352.0, + "69": 733860352.0, + "70": 733860352.0, + "71": 733860352.0, + "72": 733860352.0, + "73": 733860352.0, + "74": 733860352.0, + "75": 733860352.0, + "76": 733860352.0, + "77": 733860352.0, + "78": 733860352.0, + "79": 733860352.0, + "80": 733860352.0, + "81": 733860352.0, + "82": 733860352.0, + "83": 733860352.0, + "84": 733860352.0, + "85": 733860352.0, + "86": 733860352.0, + "87": 733860352.0, + "88": 733860352.0, + "89": 733860352.0, + "90": 733860352.0, + "91": 733860352.0, + "92": 733860352.0, + "93": 733860352.0, + "94": 733860352.0, + "95": 733860352.0, + "96": 733860352.0, + "97": 733860352.0, + "98": 733860352.0, + "99": 733860352.0, + "100": 733860352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2367879168.0, + "2": 2651687936.0, + "3": 2651687936.0, + "4": 2651687936.0, + "5": 2651687936.0, + "6": 2651687936.0, + "7": 2651687936.0, + "8": 2651687936.0, + "9": 2651687936.0, + "10": 2651687936.0, + "11": 2651687936.0, + "12": 2651687936.0, + "13": 2651687936.0, + "14": 2651687936.0, + "15": 2651687936.0, + "16": 2651687936.0, + "17": 2651687936.0, + "18": 2651687936.0, + "19": 2651687936.0, + "20": 2651687936.0, + "21": 2651687936.0, + "22": 2651687936.0, + "23": 2651687936.0, + "24": 2651687936.0, + "25": 2651687936.0, + "26": 2651687936.0, + "27": 2651687936.0, + "28": 2651687936.0, + "29": 2651687936.0, + "30": 2651687936.0, + "31": 2651687936.0, + "32": 2651687936.0, + "33": 2651687936.0, + "34": 2651687936.0, + "35": 2651687936.0, + "36": 2651687936.0, + "37": 2651687936.0, + "38": 2651687936.0, + "39": 2651687936.0, + "40": 2651687936.0, + "41": 2651687936.0, + "42": 2651687936.0, + "43": 2651687936.0, + "44": 2651687936.0, + "45": 2651687936.0, + "46": 2651687936.0, + "47": 2651687936.0, + "48": 2651687936.0, + "49": 2651687936.0, + "50": 2651687936.0, + "51": 2651687936.0, + "52": 2651687936.0, + "53": 2651687936.0, + "54": 2651687936.0, + "55": 2651687936.0, + "56": 2651687936.0, + "57": 2651687936.0, + "58": 2651687936.0, + "59": 2651687936.0, + "60": 2651687936.0, + "61": 2651687936.0, + "62": 2651687936.0, + "63": 2651687936.0, + "64": 2651687936.0, + "65": 2651687936.0, + "66": 2651687936.0, + "67": 2651687936.0, + "68": 2651687936.0, + "69": 2651687936.0, + "70": 2651687936.0, + "71": 2651687936.0, + "72": 2651687936.0, + "73": 2651687936.0, + "74": 2651687936.0, + "75": 2651687936.0, + "76": 2651687936.0, + "77": 2651687936.0, + "78": 2651687936.0, + "79": 2651687936.0, + "80": 2651687936.0, + "81": 2651687936.0, + "82": 2651687936.0, + "83": 2651687936.0, + "84": 2651687936.0, + "85": 2651687936.0, + "86": 2651687936.0, + "87": 2651687936.0, + "88": 2651687936.0, + "89": 2651687936.0, + "90": 2651687936.0, + "91": 2651687936.0, + "92": 2651687936.0, + "93": 2651687936.0, + "94": 2651687936.0, + "95": 2651687936.0, + "96": 2651687936.0, + "97": 2651687936.0, + "98": 2651687936.0, + "99": 2651687936.0, + "100": 2651687936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.6402, + "2": 0.15932, + "3": 0.13183, + "4": 0.12969, + "5": 0.12913, + "6": 0.12942, + "7": 0.12823, + "8": 0.13014, + "9": 0.1305, + "10": 0.13004, + "11": 0.12983, + "12": 0.12943, + "13": 0.12925, + "14": 0.13022, + "15": 0.12947, + "16": 0.12988, + "17": 0.12984, + "18": 0.12989, + "19": 0.12987, + "20": 0.12935, + "21": 0.12974, + "22": 0.12965, + "23": 0.12983, + "24": 0.13037, + "25": 0.1293, + "26": 0.12914, + "27": 0.12908, + "28": 0.12909, + "29": 0.13186, + "30": 0.13433, + "31": 0.13401, + "32": 0.12902, + "33": 0.12808, + "34": 0.12907, + "35": 0.12884, + "36": 0.12913, + "37": 0.12932, + "38": 0.12992, + "39": 0.13072, + "40": 0.13131, + "41": 0.13172, + "42": 0.13072, + "43": 0.13259, + "44": 0.13124, + "45": 0.13129, + "46": 0.1291, + "47": 0.1308, + "48": 0.1301, + "49": 0.12906, + "50": 0.12828, + "51": 0.14265, + "52": 0.12979, + "53": 0.126, + "54": 0.12545, + "55": 0.12582, + "56": 0.12573, + "57": 0.12516, + "58": 0.1252, + "59": 0.12598, + "60": 0.12562, + "61": 0.12544, + "62": 0.12472, + "63": 0.12548, + "64": 0.12537, + "65": 0.12534, + "66": 0.12474, + "67": 0.12528, + "68": 0.12481, + "69": 0.12531, + "70": 0.12547, + "71": 0.12492, + "72": 0.12533, + "73": 0.12583, + "74": 0.1253, + "75": 0.12453, + "76": 0.12486, + "77": 0.12501, + "78": 0.12491, + "79": 0.12247, + "80": 0.1223, + "81": 0.1243, + "82": 0.12257, + "83": 0.12179, + "84": 0.12254, + "85": 0.12231, + "86": 0.12263, + "87": 0.12152, + "88": 0.12188, + "89": 0.1228, + "90": 0.12133, + "91": 0.1216, + "92": 0.12133, + "93": 0.12135, + "94": 0.12216, + "95": 0.12141, + "96": 0.12205, + "97": 0.12356, + "98": 0.12174, + "99": 0.12252, + "100": 0.1222 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..9798744d286 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 520868352.0, + "2": 520868352.0, + "3": 520868352.0, + "4": 520868352.0, + "5": 520868352.0, + "6": 520868352.0, + "7": 520868352.0, + "8": 520868352.0, + "9": 520868352.0, + "10": 520868352.0, + "11": 520868352.0, + "12": 520868352.0, + "13": 520868352.0, + "14": 520868352.0, + "15": 520868352.0, + "16": 520868352.0, + "17": 520868352.0, + "18": 520868352.0, + "19": 520868352.0, + "20": 520868352.0, + "21": 520868352.0, + "22": 520868352.0, + "23": 520868352.0, + "24": 520868352.0, + "25": 520868352.0, + "26": 520868352.0, + "27": 520868352.0, + "28": 520868352.0, + "29": 520868352.0, + "30": 520868352.0, + "31": 520868352.0, + "32": 520868352.0, + "33": 520868352.0, + "34": 520868352.0, + "35": 520868352.0, + "36": 520868352.0, + "37": 520868352.0, + "38": 520868352.0, + "39": 520868352.0, + "40": 520868352.0, + "41": 520868352.0, + "42": 520868352.0, + "43": 520868352.0, + "44": 520868352.0, + "45": 520868352.0, + "46": 520868352.0, + "47": 520868352.0, + "48": 520868352.0, + "49": 520868352.0, + "50": 520868352.0, + "51": 520868352.0, + "52": 520868352.0, + "53": 520868352.0, + "54": 520868352.0, + "55": 520868352.0, + "56": 520868352.0, + "57": 520868352.0, + "58": 520868352.0, + "59": 520868352.0, + "60": 520868352.0, + "61": 520868352.0, + "62": 520868352.0, + "63": 520868352.0, + "64": 520868352.0, + "65": 520868352.0, + "66": 520868352.0, + "67": 520868352.0, + "68": 520868352.0, + "69": 520868352.0, + "70": 520868352.0, + "71": 520868352.0, + "72": 520868352.0, + "73": 520868352.0, + "74": 520868352.0, + "75": 520868352.0, + "76": 520868352.0, + "77": 520868352.0, + "78": 520868352.0, + "79": 520868352.0, + "80": 520868352.0, + "81": 520868352.0, + "82": 520868352.0, + "83": 520868352.0, + "84": 520868352.0, + "85": 520868352.0, + "86": 520868352.0, + "87": 520868352.0, + "88": 520868352.0, + "89": 520868352.0, + "90": 520868352.0, + "91": 520868352.0, + "92": 520868352.0, + "93": 520868352.0, + "94": 520868352.0, + "95": 520868352.0, + "96": 520868352.0, + "97": 520868352.0, + "98": 520868352.0, + "99": 520868352.0, + "100": 520868352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2297829376.0, + "2": 2439744512.0, + "3": 2439744512.0, + "4": 2439744512.0, + "5": 2439744512.0, + "6": 2439744512.0, + "7": 2439744512.0, + "8": 2439744512.0, + "9": 2439744512.0, + "10": 2439744512.0, + "11": 2439744512.0, + "12": 2439744512.0, + "13": 2439744512.0, + "14": 2439744512.0, + "15": 2439744512.0, + "16": 2439744512.0, + "17": 2439744512.0, + "18": 2439744512.0, + "19": 2439744512.0, + "20": 2439744512.0, + "21": 2439744512.0, + "22": 2439744512.0, + "23": 2439744512.0, + "24": 2439744512.0, + "25": 2439744512.0, + "26": 2439744512.0, + "27": 2439744512.0, + "28": 2439744512.0, + "29": 2439744512.0, + "30": 2439744512.0, + "31": 2439744512.0, + "32": 2439744512.0, + "33": 2439744512.0, + "34": 2439744512.0, + "35": 2439744512.0, + "36": 2439744512.0, + "37": 2439744512.0, + "38": 2439744512.0, + "39": 2439744512.0, + "40": 2439744512.0, + "41": 2439744512.0, + "42": 2439744512.0, + "43": 2439744512.0, + "44": 2439744512.0, + "45": 2439744512.0, + "46": 2439744512.0, + "47": 2439744512.0, + "48": 2439744512.0, + "49": 2439744512.0, + "50": 2439744512.0, + "51": 2439744512.0, + "52": 2439744512.0, + "53": 2439744512.0, + "54": 2439744512.0, + "55": 2439744512.0, + "56": 2439744512.0, + "57": 2439744512.0, + "58": 2439744512.0, + "59": 2439744512.0, + "60": 2439744512.0, + "61": 2439744512.0, + "62": 2439744512.0, + "63": 2439744512.0, + "64": 2439744512.0, + "65": 2439744512.0, + "66": 2439744512.0, + "67": 2439744512.0, + "68": 2439744512.0, + "69": 2439744512.0, + "70": 2439744512.0, + "71": 2439744512.0, + "72": 2439744512.0, + "73": 2439744512.0, + "74": 2439744512.0, + "75": 2439744512.0, + "76": 2439744512.0, + "77": 2439744512.0, + "78": 2439744512.0, + "79": 2439744512.0, + "80": 2439744512.0, + "81": 2439744512.0, + "82": 2439744512.0, + "83": 2439744512.0, + "84": 2439744512.0, + "85": 2439744512.0, + "86": 2439744512.0, + "87": 2439744512.0, + "88": 2439744512.0, + "89": 2439744512.0, + "90": 2439744512.0, + "91": 2439744512.0, + "92": 2439744512.0, + "93": 2439744512.0, + "94": 2439744512.0, + "95": 2439744512.0, + "96": 2439744512.0, + "97": 2439744512.0, + "98": 2439744512.0, + "99": 2439744512.0, + "100": 2439744512.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 13.16911, + "2": 0.18867, + "3": 0.13882, + "4": 0.13669, + "5": 0.13645, + "6": 0.13563, + "7": 0.13741, + "8": 0.13506, + "9": 0.13477, + "10": 0.13539, + "11": 0.13546, + "12": 0.13513, + "13": 0.13494, + "14": 0.13581, + "15": 0.13521, + "16": 0.13419, + "17": 0.14048, + "18": 0.13853, + "19": 0.13344, + "20": 0.13345, + "21": 0.13394, + "22": 0.1343, + "23": 0.13285, + "24": 0.13319, + "25": 0.13222, + "26": 0.13344, + "27": 0.13252, + "28": 0.13239, + "29": 0.1336, + "30": 0.13332, + "31": 0.13361, + "32": 0.13239, + "33": 0.13272, + "34": 0.13285, + "35": 0.21759, + "36": 0.1331, + "37": 0.1319, + "38": 0.13293, + "39": 0.13316, + "40": 0.13306, + "41": 0.13419, + "42": 0.13323, + "43": 0.13347, + "44": 0.13313, + "45": 0.13281, + "46": 0.13654, + "47": 0.13861, + "48": 0.14195, + "49": 0.14972, + "50": 0.14396, + "51": 0.1402, + "52": 0.13309, + "53": 0.1319, + "54": 0.13165, + "55": 0.1313, + "56": 0.13223, + "57": 0.13257, + "58": 0.13107, + "59": 0.13218, + "60": 0.13223, + "61": 0.13169, + "62": 0.13257, + "63": 0.13313, + "64": 0.13312, + "65": 0.1329, + "66": 0.13333, + "67": 0.13273, + "68": 0.13284, + "69": 0.1315, + "70": 0.13199, + "71": 0.13227, + "72": 0.13324, + "73": 0.13323, + "74": 0.13184, + "75": 0.13306, + "76": 0.13246, + "77": 0.13205, + "78": 0.1314, + "79": 0.13214, + "80": 0.13251, + "81": 0.1324, + "82": 0.13223, + "83": 0.133, + "84": 0.13175, + "85": 0.1338, + "86": 0.13446, + "87": 0.13241, + "88": 0.13183, + "89": 0.13223, + "90": 0.13104, + "91": 0.13159, + "92": 0.13112, + "93": 0.13189, + "94": 0.13213, + "95": 0.13312, + "96": 0.13243, + "97": 0.21757, + "98": 0.1322, + "99": 0.13344, + "100": 0.13353 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json index 85ab75a97bf..5d20ab395ec 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69212, "25": 10.52413, "30": 10.34552, "35": 10.26241, "40": 10.07236, "45": 9.81098, "50": 9.88418, "55": 9.86374, "60": 9.47963, "65": 8.93065, "70": 9.71218, "75": 9.4005, "80": 9.39078, "85": 9.60138, "90": 9.80502, "95": 9.50821, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1621.0, "25": 1689.0, "30": 1907.0, "35": 1922.0, "40": 2045.0, "45": 2097.0, "50": 2368.0, "55": 2383.0, "60": 2509.0, "65": 2628.0, "70": 3129.0, "75": 2552.0, "80": 3279.0, "85": 3344.0, "90": 2980.0, "95": 3324.0, "100": 3467.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522989568.0, "5": 522989568.0, "10": 522989568.0, "15": 522989568.0, "20": 522989568.0, "25": 522989568.0, "30": 522989568.0, "35": 522989568.0, "40": 522989568.0, "45": 522989568.0, "50": 522989568.0, "55": 522989568.0, "60": 522989568.0, "65": 522989568.0, "70": 522989568.0, "75": 522989568.0, "80": 522989568.0, "85": 522989568.0, "90": 522989568.0, "95": 522989568.0, "100": 522989568.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3768872960.0, "5": 3911833600.0, "10": 3911833600.0, "15": 3911833600.0, "20": 3911833600.0, "25": 3911833600.0, "30": 3911833600.0, "35": 3911833600.0, "40": 3911833600.0, "45": 3911833600.0, "50": 3911833600.0, "55": 3911833600.0, "60": 3911833600.0, "65": 3911833600.0, "70": 3911833600.0, "75": 3911833600.0, "80": 3911833600.0, "85": 3911833600.0, "90": 3911833600.0, "95": 3911833600.0, "100": 3911833600.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 18.37086, "5": 0.17677, "10": 0.17771, "15": 0.1763, "20": 0.17346, "25": 0.17343, "30": 0.17441, "35": 0.17427, "40": 0.17562, "45": 0.1738, "50": 0.17384, "55": 0.17214, "60": 0.17159, "65": 0.17227, "70": 0.17173, "75": 0.17264, "80": 0.1723, "85": 0.1716, "90": 0.17085, "95": 0.17315, "100": 0.1716}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 521400832.0, + "2": 521400832.0, + "3": 521400832.0, + "4": 521400832.0, + "5": 521400832.0, + "6": 521400832.0, + "7": 521400832.0, + "8": 521400832.0, + "9": 521400832.0, + "10": 521400832.0, + "11": 521400832.0, + "12": 521400832.0, + "13": 521400832.0, + "14": 521400832.0, + "15": 521400832.0, + "16": 521400832.0, + "17": 521400832.0, + "18": 521400832.0, + "19": 521400832.0, + "20": 521400832.0, + "21": 521400832.0, + "22": 521400832.0, + "23": 521400832.0, + "24": 521400832.0, + "25": 521400832.0, + "26": 521400832.0, + "27": 521400832.0, + "28": 521400832.0, + "29": 521400832.0, + "30": 521400832.0, + "31": 521400832.0, + "32": 521400832.0, + "33": 521400832.0, + "34": 521400832.0, + "35": 521400832.0, + "36": 521400832.0, + "37": 521400832.0, + "38": 521400832.0, + "39": 521400832.0, + "40": 521400832.0, + "41": 521400832.0, + "42": 521400832.0, + "43": 521400832.0, + "44": 521400832.0, + "45": 521400832.0, + "46": 521400832.0, + "47": 521400832.0, + "48": 521400832.0, + "49": 521400832.0, + "50": 521400832.0, + "51": 521400832.0, + "52": 521400832.0, + "53": 521400832.0, + "54": 521400832.0, + "55": 521400832.0, + "56": 521400832.0, + "57": 521400832.0, + "58": 521400832.0, + "59": 521400832.0, + "60": 521400832.0, + "61": 521400832.0, + "62": 521400832.0, + "63": 521400832.0, + "64": 521400832.0, + "65": 521400832.0, + "66": 521400832.0, + "67": 521400832.0, + "68": 521400832.0, + "69": 521400832.0, + "70": 521400832.0, + "71": 521400832.0, + "72": 521400832.0, + "73": 521400832.0, + "74": 521400832.0, + "75": 521400832.0, + "76": 521400832.0, + "77": 521400832.0, + "78": 521400832.0, + "79": 521400832.0, + "80": 521400832.0, + "81": 521400832.0, + "82": 521400832.0, + "83": 521400832.0, + "84": 521400832.0, + "85": 521400832.0, + "86": 521400832.0, + "87": 521400832.0, + "88": 521400832.0, + "89": 521400832.0, + "90": 521400832.0, + "91": 521400832.0, + "92": 521400832.0, + "93": 521400832.0, + "94": 521400832.0, + "95": 521400832.0, + "96": 521400832.0, + "97": 521400832.0, + "98": 521400832.0, + "99": 521400832.0, + "100": 521400832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2297829376.0, + "2": 2439228416.0, + "3": 2439228416.0, + "4": 2439228416.0, + "5": 2439228416.0, + "6": 2439228416.0, + "7": 2439228416.0, + "8": 2439228416.0, + "9": 2439228416.0, + "10": 2439228416.0, + "11": 2439228416.0, + "12": 2439228416.0, + "13": 2439228416.0, + "14": 2439228416.0, + "15": 2439228416.0, + "16": 2439228416.0, + "17": 2439228416.0, + "18": 2439228416.0, + "19": 2439228416.0, + "20": 2439228416.0, + "21": 2439228416.0, + "22": 2439228416.0, + "23": 2439228416.0, + "24": 2439228416.0, + "25": 2439228416.0, + "26": 2439228416.0, + "27": 2439228416.0, + "28": 2439228416.0, + "29": 2439228416.0, + "30": 2439228416.0, + "31": 2439228416.0, + "32": 2439228416.0, + "33": 2439228416.0, + "34": 2439228416.0, + "35": 2439228416.0, + "36": 2439228416.0, + "37": 2439228416.0, + "38": 2439228416.0, + "39": 2439228416.0, + "40": 2439228416.0, + "41": 2439228416.0, + "42": 2439228416.0, + "43": 2439228416.0, + "44": 2439228416.0, + "45": 2439228416.0, + "46": 2439228416.0, + "47": 2439228416.0, + "48": 2439228416.0, + "49": 2439228416.0, + "50": 2439228416.0, + "51": 2439228416.0, + "52": 2439228416.0, + "53": 2439228416.0, + "54": 2439228416.0, + "55": 2439228416.0, + "56": 2439228416.0, + "57": 2439228416.0, + "58": 2439228416.0, + "59": 2439228416.0, + "60": 2439228416.0, + "61": 2439228416.0, + "62": 2439228416.0, + "63": 2439228416.0, + "64": 2439228416.0, + "65": 2439228416.0, + "66": 2439228416.0, + "67": 2439228416.0, + "68": 2439228416.0, + "69": 2439228416.0, + "70": 2439228416.0, + "71": 2439228416.0, + "72": 2439228416.0, + "73": 2439228416.0, + "74": 2439228416.0, + "75": 2439228416.0, + "76": 2439228416.0, + "77": 2439228416.0, + "78": 2439228416.0, + "79": 2439228416.0, + "80": 2439228416.0, + "81": 2439228416.0, + "82": 2439228416.0, + "83": 2439228416.0, + "84": 2439228416.0, + "85": 2439228416.0, + "86": 2439228416.0, + "87": 2439228416.0, + "88": 2439228416.0, + "89": 2439228416.0, + "90": 2439228416.0, + "91": 2439228416.0, + "92": 2439228416.0, + "93": 2439228416.0, + "94": 2439228416.0, + "95": 2439228416.0, + "96": 2439228416.0, + "97": 2439228416.0, + "98": 2439228416.0, + "99": 2439228416.0, + "100": 2439228416.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.8604, + "2": 0.16953, + "3": 0.13987, + "4": 0.13824, + "5": 0.13775, + "6": 0.13549, + "7": 0.13611, + "8": 0.13584, + "9": 0.13626, + "10": 0.13922, + "11": 0.13526, + "12": 0.13455, + "13": 0.13222, + "14": 0.13324, + "15": 0.1325, + "16": 0.13211, + "17": 0.13198, + "18": 0.13145, + "19": 0.13207, + "20": 0.13182, + "21": 0.13297, + "22": 0.1322, + "23": 0.13275, + "24": 0.1319, + "25": 0.13822, + "26": 0.13214, + "27": 0.13169, + "28": 0.13196, + "29": 0.13229, + "30": 0.13285, + "31": 0.13112, + "32": 0.13222, + "33": 0.13056, + "34": 0.13076, + "35": 0.13218, + "36": 0.13126, + "37": 0.13091, + "38": 0.13048, + "39": 0.13082, + "40": 0.1308, + "41": 0.13202, + "42": 0.1314, + "43": 0.13222, + "44": 0.13074, + "45": 0.13237, + "46": 0.13272, + "47": 0.13239, + "48": 0.13266, + "49": 0.13226, + "50": 0.13164, + "51": 0.13425, + "52": 0.13044, + "53": 0.13037, + "54": 0.13007, + "55": 0.1301, + "56": 0.13001, + "57": 0.13054, + "58": 0.12972, + "59": 0.13049, + "60": 0.13042, + "61": 0.12903, + "62": 0.13042, + "63": 0.13104, + "64": 0.13008, + "65": 0.13158, + "66": 0.13091, + "67": 0.13089, + "68": 0.13084, + "69": 0.12903, + "70": 0.13015, + "71": 0.12957, + "72": 0.12997, + "73": 0.13025, + "74": 0.12989, + "75": 0.13018, + "76": 0.12962, + "77": 0.13065, + "78": 0.12915, + "79": 0.13007, + "80": 0.12972, + "81": 0.1301, + "82": 0.12927, + "83": 0.1302, + "84": 0.12991, + "85": 0.13129, + "86": 0.13063, + "87": 0.13028, + "88": 0.1305, + "89": 0.13046, + "90": 0.12991, + "91": 0.13058, + "92": 0.13044, + "93": 0.13009, + "94": 0.1306, + "95": 0.13082, + "96": 0.13068, + "97": 0.13403, + "98": 0.13199, + "99": 0.13191, + "100": 0.13014 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..0532ff11573 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 14.21193, + "2": 0.16972, + "3": 0.14361, + "4": 0.13668, + "5": 0.13411, + "6": 0.13482, + "7": 0.13456, + "8": 0.13436, + "9": 0.13229, + "10": 0.13468, + "11": 0.29227, + "12": 0.13377, + "13": 0.13424, + "14": 0.1347, + "15": 0.13353, + "16": 0.13405, + "17": 0.13386, + "18": 0.13335, + "19": 0.1337, + "20": 0.13441, + "21": 0.28803, + "22": 0.13435, + "23": 0.13391, + "24": 0.1344, + "25": 0.13379, + "26": 0.13318, + "27": 0.13391, + "28": 0.13299, + "29": 0.13332, + "30": 0.13439, + "31": 0.28662, + "32": 0.13342, + "33": 0.1334, + "34": 0.13306, + "35": 0.13383, + "36": 0.13334, + "37": 0.1329, + "38": 0.13424, + "39": 0.13255, + "40": 0.13274, + "41": 0.28399, + "42": 0.13366, + "43": 0.13392, + "44": 0.13317, + "45": 0.13301, + "46": 0.13375, + "47": 0.13359, + "48": 0.13309, + "49": 0.13333, + "50": 0.13346, + "51": 0.28956, + "52": 0.14009, + "53": 0.13828, + "54": 0.1377, + "55": 0.13637, + "56": 0.13795, + "57": 0.13737, + "58": 0.13417, + "59": 0.13327, + "60": 0.13391, + "61": 0.29272, + "62": 0.13275, + "63": 0.13346, + "64": 0.13336, + "65": 0.13291, + "66": 0.13365, + "67": 0.13307, + "68": 0.13243, + "69": 0.1327, + "70": 0.13323, + "71": 0.28929, + "72": 0.1323, + "73": 0.13407, + "74": 0.13317, + "75": 0.132, + "76": 0.13278, + "77": 0.13359, + "78": 0.13257, + "79": 0.13226, + "80": 0.13339, + "81": 0.28457, + "82": 0.13147, + "83": 0.13289, + "84": 0.13213, + "85": 0.13375, + "86": 0.13168, + "87": 0.13351, + "88": 0.13059, + "89": 0.13189, + "90": 0.13102, + "91": 0.28755, + "92": 0.13136, + "93": 0.13235, + "94": 0.13175, + "95": 0.13072, + "96": 0.13269, + "97": 0.13234, + "98": 0.1311, + "99": 0.13293, + "100": 0.13016 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index cda47f321c8..e45c3949555 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69212, "25": 10.52413, "30": 10.34552, "35": 10.26241, "40": 10.07236, "45": 9.81098, "50": 9.88418, "55": 9.86374, "60": 9.47963, "65": 8.93065, "70": 9.71218, "75": 9.4005, "80": 9.39078, "85": 9.60138, "90": 9.80502, "95": 9.50821, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1621.0, "25": 1689.0, "30": 1907.0, "35": 1922.0, "40": 2045.0, "45": 2097.0, "50": 2368.0, "55": 2383.0, "60": 2509.0, "65": 2628.0, "70": 3129.0, "75": 2552.0, "80": 3279.0, "85": 3344.0, "90": 2980.0, "95": 3324.0, "100": 3467.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522989568.0, "5": 522989568.0, "10": 522989568.0, "15": 522989568.0, "20": 522989568.0, "25": 522989568.0, "30": 522989568.0, "35": 522989568.0, "40": 522989568.0, "45": 522989568.0, "50": 522989568.0, "55": 522989568.0, "60": 522989568.0, "65": 522989568.0, "70": 522989568.0, "75": 522989568.0, "80": 522989568.0, "85": 522989568.0, "90": 522989568.0, "95": 522989568.0, "100": 522989568.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3768872960.0, "5": 3911833600.0, "10": 3911833600.0, "15": 3911833600.0, "20": 3911833600.0, "25": 3911833600.0, "30": 3911833600.0, "35": 3911833600.0, "40": 3911833600.0, "45": 3911833600.0, "50": 3911833600.0, "55": 3911833600.0, "60": 3911833600.0, "65": 3911833600.0, "70": 3911833600.0, "75": 3911833600.0, "80": 3911833600.0, "85": 3911833600.0, "90": 3911833600.0, "95": 3911833600.0, "100": 3911833600.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 24.27643, "5": 0.18328, "10": 0.18099, "15": 0.17942, "20": 0.17956, "25": 0.1771, "30": 0.17799, "35": 0.17821, "40": 0.1777, "45": 0.17833, "50": 0.17766, "55": 0.17789, "60": 0.17744, "65": 0.17689, "70": 0.1767, "75": 0.17717, "80": 0.17685, "85": 0.17672, "90": 0.17477, "95": 0.17583, "100": 0.17624}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.90397, + "2": 0.16607, + "3": 0.13982, + "4": 0.14032, + "5": 0.13765, + "6": 0.13651, + "7": 0.13453, + "8": 0.13413, + "9": 0.13703, + "10": 0.13873, + "11": 0.28364, + "12": 0.13723, + "13": 0.13756, + "14": 0.1379, + "15": 0.14148, + "16": 0.1356, + "17": 0.13661, + "18": 0.13568, + "19": 0.13637, + "20": 0.1367, + "21": 0.28276, + "22": 0.13722, + "23": 0.13404, + "24": 0.13414, + "25": 0.1341, + "26": 0.13595, + "27": 0.13446, + "28": 0.13477, + "29": 0.13439, + "30": 0.13383, + "31": 0.27955, + "32": 0.13416, + "33": 0.13472, + "34": 0.13383, + "35": 0.13499, + "36": 0.13468, + "37": 0.13332, + "38": 0.13449, + "39": 0.13488, + "40": 0.1347, + "41": 0.2818, + "42": 0.13497, + "43": 0.13495, + "44": 0.13372, + "45": 0.13385, + "46": 0.13479, + "47": 0.13339, + "48": 0.13334, + "49": 0.13393, + "50": 0.13346, + "51": 0.2815, + "52": 0.13492, + "53": 0.13387, + "54": 0.13407, + "55": 0.13263, + "56": 0.13379, + "57": 0.13439, + "58": 0.13407, + "59": 0.13481, + "60": 0.13407, + "61": 0.28073, + "62": 0.13474, + "63": 0.13363, + "64": 0.13359, + "65": 0.13323, + "66": 0.13437, + "67": 0.13391, + "68": 0.13344, + "69": 0.21561, + "70": 0.1337, + "71": 0.27778, + "72": 0.13359, + "73": 0.13364, + "74": 0.13406, + "75": 0.13376, + "76": 0.13308, + "77": 0.13263, + "78": 0.13172, + "79": 0.13328, + "80": 0.13387, + "81": 0.28018, + "82": 0.13437, + "83": 0.13645, + "84": 0.13548, + "85": 0.13558, + "86": 0.13447, + "87": 0.13492, + "88": 0.13361, + "89": 0.13427, + "90": 0.13332, + "91": 0.27771, + "92": 0.13375, + "93": 0.1331, + "94": 0.13317, + "95": 0.13408, + "96": 0.13418, + "97": 0.13752, + "98": 0.13493, + "99": 0.13408, + "100": 0.13136 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json new file mode 100644 index 00000000000..6f64205e3c4 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93693, + "2": 10.92078, + "3": 10.92311, + "4": 10.908, + "5": 10.93133, + "6": 10.93476, + "7": 10.92722, + "8": 10.91987, + "9": 10.93801, + "10": 10.91377, + "11": 10.94194, + "12": 10.93222, + "13": 10.92553, + "14": 10.94301, + "15": 10.86288, + "16": 10.87945, + "17": 10.86904, + "18": 10.88527, + "19": 10.86912, + "20": 10.77336, + "21": 10.75658, + "22": 10.62975, + "23": 10.76834, + "24": 10.65017, + "25": 10.60362, + "26": 10.66269, + "27": 10.66488, + "28": 10.60369, + "29": 10.64791, + "30": 10.40493, + "31": 10.16869, + "32": 10.5139, + "33": 10.50484, + "34": 10.27115, + "35": 10.31433, + "36": 10.27029, + "37": 10.38626, + "38": 10.23175, + "39": 10.45527, + "40": 10.12357, + "41": 10.19645, + "42": 10.25476, + "43": 9.86653, + "44": 9.99586, + "45": 9.87497, + "46": 9.86189, + "47": 10.19545, + "48": 9.87912, + "49": 9.56741, + "50": 9.94519, + "51": 9.89774, + "52": 9.78773, + "53": 10.12739, + "54": 9.98265, + "55": 9.90115, + "56": 9.66568, + "57": 9.49996, + "58": 9.87635, + "59": 9.61861, + "60": 9.55292, + "61": 9.71598, + "62": 10.03451, + "63": 9.41294, + "64": 9.81949, + "65": 8.96909, + "66": 9.7478, + "67": 9.39393, + "68": 9.82085, + "69": 9.82417, + "70": 9.77915, + "71": 9.64756, + "72": 9.59555, + "73": 9.53704, + "74": 8.96643, + "75": 9.44556, + "76": 9.09922, + "77": 10.10009, + "78": 9.73576, + "79": 9.38721, + "80": 9.41961, + "81": 9.50968, + "82": 9.72303, + "83": 9.34684, + "84": 9.44855, + "85": 9.64312, + "86": 9.07818, + "87": 9.61561, + "88": 9.79072, + "89": 9.61878, + "90": 9.85929, + "91": 9.35347, + "92": 9.38799, + "93": 9.07947, + "94": 8.82329, + "95": 9.52141, + "96": 9.54598, + "97": 9.34241, + "98": 9.70512, + "99": 8.8917, + "100": 9.43443 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727688.0, + "2": 22924916.0, + "3": 22596752.0, + "4": 23219574.0, + "5": 22715594.0, + "6": 23021444.0, + "7": 22771554.0, + "8": 22926856.0, + "9": 22842096.0, + "10": 22918196.0, + "11": 22500760.0, + "12": 22460572.0, + "13": 22917018.0, + "14": 22389964.0, + "15": 22820720.0, + "16": 22830308.0, + "17": 22819122.0, + "18": 22582876.0, + "19": 22617912.0, + "20": 22693480.0, + "21": 22740412.0, + "22": 22799838.0, + "23": 22539332.0, + "24": 22770580.0, + "25": 22818910.0, + "26": 22547838.0, + "27": 22468232.0, + "28": 22452766.0, + "29": 22528906.0, + "30": 22631718.0, + "31": 22955520.0, + "32": 22584888.0, + "33": 22558558.0, + "34": 22836286.0, + "35": 22788956.0, + "36": 22589940.0, + "37": 22496912.0, + "38": 22897132.0, + "39": 22801712.0, + "40": 22657514.0, + "41": 22659596.0, + "42": 22667300.0, + "43": 22976308.0, + "44": 22745816.0, + "45": 22675136.0, + "46": 22884540.0, + "47": 22633726.0, + "48": 22928020.0, + "49": 22727656.0, + "50": 22905162.0, + "51": 22791660.0, + "52": 22748630.0, + "53": 22925420.0, + "54": 22839176.0, + "55": 22518792.0, + "56": 22877648.0, + "57": 23113592.0, + "58": 22845136.0, + "59": 22715486.0, + "60": 22743726.0, + "61": 22724104.0, + "62": 22673746.0, + "63": 22846740.0, + "64": 22823862.0, + "65": 23061360.0, + "66": 22729628.0, + "67": 22907694.0, + "68": 22609888.0, + "69": 22584610.0, + "70": 22829190.0, + "71": 22749038.0, + "72": 22655052.0, + "73": 22739796.0, + "74": 23047606.0, + "75": 23054054.0, + "76": 22901052.0, + "77": 22271520.0, + "78": 22788892.0, + "79": 22743418.0, + "80": 22706694.0, + "81": 22890786.0, + "82": 22777316.0, + "83": 22839636.0, + "84": 23010088.0, + "85": 22712948.0, + "86": 23103380.0, + "87": 22735258.0, + "88": 22636824.0, + "89": 22498828.0, + "90": 22972694.0, + "91": 22767128.0, + "92": 22808892.0, + "93": 22659368.0, + "94": 22911112.0, + "95": 23047740.0, + "96": 22828678.0, + "97": 22608174.0, + "98": 22762982.0, + "99": 22905800.0, + "100": 23015792.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.56211, + "2": 0.17032, + "3": 0.14703, + "4": 0.1399, + "5": 0.13723, + "6": 0.13231, + "7": 0.13564, + "8": 0.13449, + "9": 0.13358, + "10": 0.13307, + "11": 0.13725, + "12": 0.13165, + "13": 0.13322, + "14": 0.13288, + "15": 0.13324, + "16": 0.13281, + "17": 0.13322, + "18": 0.13237, + "19": 0.13164, + "20": 0.13184, + "21": 0.13286, + "22": 0.13291, + "23": 0.13443, + "24": 0.14138, + "25": 0.13774, + "26": 0.13805, + "27": 0.13868, + "28": 0.13766, + "29": 0.1395, + "30": 0.13538, + "31": 0.13589, + "32": 0.13529, + "33": 0.13562, + "34": 0.13525, + "35": 0.13556, + "36": 0.13544, + "37": 0.13549, + "38": 0.13569, + "39": 0.13558, + "40": 0.13561, + "41": 0.13653, + "42": 0.13569, + "43": 0.13645, + "44": 0.13586, + "45": 0.1354, + "46": 0.13606, + "47": 0.13517, + "48": 0.13615, + "49": 0.13593, + "50": 0.13488, + "51": 0.13643, + "52": 0.13115, + "53": 0.13291, + "54": 0.13115, + "55": 0.13085, + "56": 0.1309, + "57": 0.13094, + "58": 0.13092, + "59": 0.13092, + "60": 0.13094, + "61": 0.13102, + "62": 0.13092, + "63": 0.13052, + "64": 0.1313, + "65": 0.13151, + "66": 0.1314, + "67": 0.13094, + "68": 0.1308, + "69": 0.13079, + "70": 0.1309, + "71": 0.13083, + "72": 0.13083, + "73": 0.13091, + "74": 0.13061, + "75": 0.13042, + "76": 0.13145, + "77": 0.13079, + "78": 0.13108, + "79": 0.13106, + "80": 0.13055, + "81": 0.13075, + "82": 0.13043, + "83": 0.13151, + "84": 0.13173, + "85": 0.13126, + "86": 0.13187, + "87": 0.13084, + "88": 0.13098, + "89": 0.13086, + "90": 0.13054, + "91": 0.1317, + "92": 0.13098, + "93": 0.13084, + "94": 0.13088, + "95": 0.13054, + "96": 0.13148, + "97": 0.13154, + "98": 0.13179, + "99": 0.13086, + "100": 0.13121 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json index e57b1a644a0..7848ef42dd8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.9359, "5": 10.93225, "10": 10.91083, "15": 10.85723, "20": 10.77088, "25": 10.60556, "30": 10.40549, "35": 10.31366, "40": 10.12334, "45": 9.87564, "50": 9.94452, "55": 9.90094, "60": 9.55238, "65": 8.96792, "70": 9.77835, "75": 9.44605, "80": 9.42014, "85": 9.64324, "90": 9.85827, "95": 9.52085, "100": 9.43415}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 22727686.0, "5": 22715312.0, "10": 22919060.0, "15": 22821200.0, "20": 22693840.0, "25": 22819524.0, "30": 22631232.0, "35": 22787888.0, "40": 22658144.0, "45": 22674630.0, "50": 22904452.0, "55": 22519094.0, "60": 22743052.0, "65": 23061016.0, "70": 22829332.0, "75": 23054164.0, "80": 22707344.0, "85": 22712182.0, "90": 22971846.0, "95": 23047676.0, "100": 23015938.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522900480.0, "5": 522900480.0, "10": 522900480.0, "15": 522900480.0, "20": 522900480.0, "25": 522900480.0, "30": 522900480.0, "35": 522900480.0, "40": 522900480.0, "45": 522900480.0, "50": 522900480.0, "55": 522900480.0, "60": 522900480.0, "65": 522900480.0, "70": 522900480.0, "75": 522900480.0, "80": 522900480.0, "85": 522900480.0, "90": 522900480.0, "95": 522900480.0, "100": 522900480.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3769943040.0, "5": 3912677376.0, "10": 3912677376.0, "15": 3912677376.0, "20": 3912677376.0, "25": 3912677376.0, "30": 3912677376.0, "35": 3912677376.0, "40": 3912677376.0, "45": 3912677376.0, "50": 3912677376.0, "55": 3912677376.0, "60": 3912677376.0, "65": 3912677376.0, "70": 3912677376.0, "75": 3912677376.0, "80": 3912677376.0, "85": 3912677376.0, "90": 3912677376.0, "95": 3912677376.0, "100": 3912677376.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 22.64226, "5": 0.17675, "10": 0.17674, "15": 0.17989, "20": 0.17736, "25": 0.1752, "30": 0.17793, "35": 0.17672, "40": 0.17711, "45": 0.17664, "50": 0.17313, "55": 0.17519, "60": 0.17937, "65": 0.17909, "70": 0.17798, "75": 0.17301, "80": 0.17354, "85": 0.17313, "90": 0.1739, "95": 0.17379, "100": 0.17144}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93693, + "2": 10.92078, + "3": 10.92311, + "4": 10.908, + "5": 10.93133, + "6": 10.93476, + "7": 10.92722, + "8": 10.91987, + "9": 10.93801, + "10": 10.91377, + "11": 10.94194, + "12": 10.93222, + "13": 10.92553, + "14": 10.94301, + "15": 10.86288, + "16": 10.87945, + "17": 10.86904, + "18": 10.88527, + "19": 10.86912, + "20": 10.77336, + "21": 10.75658, + "22": 10.62975, + "23": 10.76834, + "24": 10.65017, + "25": 10.60362, + "26": 10.66269, + "27": 10.66488, + "28": 10.60369, + "29": 10.64791, + "30": 10.40493, + "31": 10.16869, + "32": 10.5139, + "33": 10.50484, + "34": 10.27115, + "35": 10.31433, + "36": 10.27029, + "37": 10.38626, + "38": 10.23175, + "39": 10.45527, + "40": 10.12357, + "41": 10.19645, + "42": 10.25476, + "43": 9.86653, + "44": 9.99586, + "45": 9.87497, + "46": 9.86189, + "47": 10.19545, + "48": 9.87912, + "49": 9.56741, + "50": 9.94519, + "51": 9.89774, + "52": 9.78773, + "53": 10.12739, + "54": 9.98265, + "55": 9.90115, + "56": 9.66568, + "57": 9.49996, + "58": 9.87635, + "59": 9.61861, + "60": 9.55292, + "61": 9.71598, + "62": 10.03451, + "63": 9.41294, + "64": 9.81949, + "65": 8.96909, + "66": 9.7478, + "67": 9.39393, + "68": 9.82085, + "69": 9.82417, + "70": 9.77915, + "71": 9.64756, + "72": 9.59555, + "73": 9.53704, + "74": 8.96643, + "75": 9.44556, + "76": 9.09922, + "77": 10.10009, + "78": 9.73576, + "79": 9.38721, + "80": 9.41961, + "81": 9.50968, + "82": 9.72303, + "83": 9.34684, + "84": 9.44855, + "85": 9.64312, + "86": 9.07818, + "87": 9.61561, + "88": 9.79072, + "89": 9.61878, + "90": 9.85929, + "91": 9.35347, + "92": 9.38799, + "93": 9.07947, + "94": 8.82329, + "95": 9.52141, + "96": 9.54598, + "97": 9.34241, + "98": 9.70512, + "99": 8.8917, + "100": 9.43443 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727688.0, + "2": 22924916.0, + "3": 22596752.0, + "4": 23219574.0, + "5": 22715594.0, + "6": 23021444.0, + "7": 22771554.0, + "8": 22926856.0, + "9": 22842096.0, + "10": 22918196.0, + "11": 22500760.0, + "12": 22460572.0, + "13": 22917018.0, + "14": 22389964.0, + "15": 22820720.0, + "16": 22830308.0, + "17": 22819122.0, + "18": 22582876.0, + "19": 22617912.0, + "20": 22693480.0, + "21": 22740412.0, + "22": 22799838.0, + "23": 22539332.0, + "24": 22770580.0, + "25": 22818910.0, + "26": 22547838.0, + "27": 22468232.0, + "28": 22452766.0, + "29": 22528906.0, + "30": 22631718.0, + "31": 22955520.0, + "32": 22584888.0, + "33": 22558558.0, + "34": 22836286.0, + "35": 22788956.0, + "36": 22589940.0, + "37": 22496912.0, + "38": 22897132.0, + "39": 22801712.0, + "40": 22657514.0, + "41": 22659596.0, + "42": 22667300.0, + "43": 22976308.0, + "44": 22745816.0, + "45": 22675136.0, + "46": 22884540.0, + "47": 22633726.0, + "48": 22928020.0, + "49": 22727656.0, + "50": 22905162.0, + "51": 22791660.0, + "52": 22748630.0, + "53": 22925420.0, + "54": 22839176.0, + "55": 22518792.0, + "56": 22877648.0, + "57": 23113592.0, + "58": 22845136.0, + "59": 22715486.0, + "60": 22743726.0, + "61": 22724104.0, + "62": 22673746.0, + "63": 22846740.0, + "64": 22823862.0, + "65": 23061360.0, + "66": 22729628.0, + "67": 22907694.0, + "68": 22609888.0, + "69": 22584610.0, + "70": 22829190.0, + "71": 22749038.0, + "72": 22655052.0, + "73": 22739796.0, + "74": 23047606.0, + "75": 23054054.0, + "76": 22901052.0, + "77": 22271520.0, + "78": 22788892.0, + "79": 22743418.0, + "80": 22706694.0, + "81": 22890786.0, + "82": 22777316.0, + "83": 22839636.0, + "84": 23010088.0, + "85": 22712948.0, + "86": 23103380.0, + "87": 22735258.0, + "88": 22636824.0, + "89": 22498828.0, + "90": 22972694.0, + "91": 22767128.0, + "92": 22808892.0, + "93": 22659368.0, + "94": 22911112.0, + "95": 23047740.0, + "96": 22828678.0, + "97": 22608174.0, + "98": 22762982.0, + "99": 22905800.0, + "100": 23015792.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 13.44016, + "2": 0.17357, + "3": 0.14155, + "4": 0.14433, + "5": 0.14312, + "6": 0.14041, + "7": 0.14082, + "8": 0.13921, + "9": 0.1399, + "10": 0.13856, + "11": 0.13995, + "12": 0.13864, + "13": 0.13803, + "14": 0.13783, + "15": 0.13752, + "16": 0.13882, + "17": 0.13834, + "18": 0.13863, + "19": 0.13872, + "20": 0.1384, + "21": 0.13424, + "22": 0.13105, + "23": 0.13094, + "24": 0.1307, + "25": 0.13252, + "26": 0.13172, + "27": 0.12995, + "28": 0.13015, + "29": 0.13002, + "30": 0.13019, + "31": 0.13071, + "32": 0.13106, + "33": 0.1305, + "34": 0.13023, + "35": 0.13178, + "36": 0.13167, + "37": 0.13002, + "38": 0.13094, + "39": 0.13093, + "40": 0.13167, + "41": 0.13178, + "42": 0.13107, + "43": 0.1328, + "44": 0.13048, + "45": 0.13046, + "46": 0.13126, + "47": 0.12901, + "48": 0.12854, + "49": 0.12862, + "50": 0.12918, + "51": 0.14204, + "52": 0.13766, + "53": 0.13573, + "54": 0.13601, + "55": 0.13392, + "56": 0.13591, + "57": 0.13683, + "58": 0.13487, + "59": 0.13645, + "60": 0.13627, + "61": 0.13507, + "62": 0.13578, + "63": 0.13619, + "64": 0.13556, + "65": 0.13673, + "66": 0.13706, + "67": 0.13535, + "68": 0.13581, + "69": 0.1342, + "70": 0.13519, + "71": 0.13563, + "72": 0.13553, + "73": 0.13626, + "74": 0.13636, + "75": 0.1351, + "76": 0.13531, + "77": 0.1341, + "78": 0.13121, + "79": 0.13164, + "80": 0.1338, + "81": 0.13214, + "82": 0.13227, + "83": 0.13301, + "84": 0.13291, + "85": 0.13384, + "86": 0.13276, + "87": 0.13499, + "88": 0.13549, + "89": 0.13554, + "90": 0.13505, + "91": 0.13486, + "92": 0.13406, + "93": 0.13522, + "94": 0.13615, + "95": 0.1365, + "96": 0.13586, + "97": 0.13623, + "98": 0.13603, + "99": 0.13615, + "100": 0.13526 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..03ebc195862 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80831, + "2": 10.81091, + "3": 10.82881, + "4": 10.82888, + "5": 10.8518, + "6": 10.84129, + "7": 10.839, + "8": 10.80345, + "9": 10.87875, + "10": 10.88064, + "11": 10.87605, + "12": 10.82786, + "13": 10.84179, + "14": 10.81686, + "15": 10.80637, + "16": 10.79763, + "17": 10.76624, + "18": 10.78069, + "19": 10.75188, + "20": 10.63371, + "21": 10.68119, + "22": 10.63819, + "23": 10.75544, + "24": 10.61129, + "25": 10.47478, + "26": 10.59702, + "27": 10.53977, + "28": 10.45162, + "29": 10.39421, + "30": 10.39348, + "31": 10.49397, + "32": 10.32158, + "33": 10.27676, + "34": 10.44629, + "35": 9.96856, + "36": 10.11335, + "37": 10.02306, + "38": 10.37579, + "39": 9.78845, + "40": 10.10124, + "41": 10.12323, + "42": 10.02969, + "43": 10.19869, + "44": 10.05528, + "45": 9.68289, + "46": 9.98143, + "47": 9.92628, + "48": 9.66986, + "49": 9.9198, + "50": 9.9226, + "51": 9.79344, + "52": 9.32777, + "53": 9.65056, + "54": 9.8608, + "55": 9.98167, + "56": 9.81701, + "57": 9.74358, + "58": 9.83, + "59": 9.32869, + "60": 9.35336, + "61": 9.45108, + "62": 10.19054, + "63": 9.35643, + "64": 9.62865, + "65": 9.70227, + "66": 9.52515, + "67": 9.6623, + "68": 9.58775, + "69": 9.38554, + "70": 9.73858, + "71": 9.87574, + "72": 9.69164, + "73": 9.39216, + "74": 9.43911, + "75": 8.95557, + "76": 9.56378, + "77": 9.6144, + "78": 9.39189, + "79": 9.52909, + "80": 9.31414, + "81": 9.70167, + "82": 9.90384, + "83": 9.31614, + "84": 9.47152, + "85": 8.97772, + "86": 9.66417, + "87": 9.43151, + "88": 9.58677, + "89": 9.52297, + "90": 9.55832, + "91": 9.62794, + "92": 9.14014, + "93": 9.42404, + "94": 9.54555, + "95": 9.13499, + "96": 8.75216, + "97": 9.58103, + "98": 9.79018, + "99": 9.37904, + "100": 9.21212 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1004.0, + "2": 1243.0, + "3": 1236.0, + "4": 1235.0, + "5": 1179.0, + "6": 1319.0, + "7": 1173.0, + "8": 1020.0, + "9": 1205.0, + "10": 1270.0, + "11": 1163.0, + "12": 1276.0, + "13": 1398.0, + "14": 1096.0, + "15": 1148.0, + "16": 1104.0, + "17": 1141.0, + "18": 1312.0, + "19": 1200.0, + "20": 1094.0, + "21": 1197.0, + "22": 1286.0, + "23": 1353.0, + "24": 1321.0, + "25": 1053.0, + "26": 1219.0, + "27": 1300.0, + "28": 1428.0, + "29": 1300.0, + "30": 1550.0, + "31": 1500.0, + "32": 1450.0, + "33": 1597.0, + "34": 1487.0, + "35": 1196.0, + "36": 1333.0, + "37": 1457.0, + "38": 1618.0, + "39": 1321.0, + "40": 1467.0, + "41": 1743.0, + "42": 1583.0, + "43": 1726.0, + "44": 1687.0, + "45": 1607.0, + "46": 1887.0, + "47": 1694.0, + "48": 1589.0, + "49": 1754.0, + "50": 1681.0, + "51": 1792.0, + "52": 1765.0, + "53": 1874.0, + "54": 1854.0, + "55": 1816.0, + "56": 1894.0, + "57": 1855.0, + "58": 1733.0, + "59": 1425.0, + "60": 1916.0, + "61": 2236.0, + "62": 2138.0, + "63": 2016.0, + "64": 2084.0, + "65": 2473.0, + "66": 2130.0, + "67": 2275.0, + "68": 2207.0, + "69": 2199.0, + "70": 2462.0, + "71": 2390.0, + "72": 2501.0, + "73": 1957.0, + "74": 2120.0, + "75": 2095.0, + "76": 2325.0, + "77": 2395.0, + "78": 2502.0, + "79": 2700.0, + "80": 1996.0, + "81": 2404.0, + "82": 2501.0, + "83": 2493.0, + "84": 2074.0, + "85": 2167.0, + "86": 2273.0, + "87": 2540.0, + "88": 2295.0, + "89": 2583.0, + "90": 2139.0, + "91": 2390.0, + "92": 1929.0, + "93": 2179.0, + "94": 2522.0, + "95": 2468.0, + "96": 2219.0, + "97": 2145.0, + "98": 2343.0, + "99": 2330.0, + "100": 1971.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2513633792.0, + "2": 2795345408.0, + "3": 2795345408.0, + "4": 2795345408.0, + "5": 2795345408.0, + "6": 2795345408.0, + "7": 2795345408.0, + "8": 2795345408.0, + "9": 2795345408.0, + "10": 2795345408.0, + "11": 2795345408.0, + "12": 2795345408.0, + "13": 2795345408.0, + "14": 2795345408.0, + "15": 2795345408.0, + "16": 2795345408.0, + "17": 2795345408.0, + "18": 2795345408.0, + "19": 2795345408.0, + "20": 2795345408.0, + "21": 2795345408.0, + "22": 2795345408.0, + "23": 2795345408.0, + "24": 2795345408.0, + "25": 2795345408.0, + "26": 2795345408.0, + "27": 2795345408.0, + "28": 2795345408.0, + "29": 2795345408.0, + "30": 2795345408.0, + "31": 2795345408.0, + "32": 2795345408.0, + "33": 2795345408.0, + "34": 2795345408.0, + "35": 2795345408.0, + "36": 2795345408.0, + "37": 2795345408.0, + "38": 2795345408.0, + "39": 2795345408.0, + "40": 2795345408.0, + "41": 2795345408.0, + "42": 2795345408.0, + "43": 2795345408.0, + "44": 2795345408.0, + "45": 2795345408.0, + "46": 2795345408.0, + "47": 2795345408.0, + "48": 2795345408.0, + "49": 2795345408.0, + "50": 2795345408.0, + "51": 2795345408.0, + "52": 2795345408.0, + "53": 2795345408.0, + "54": 2795345408.0, + "55": 2795345408.0, + "56": 2795345408.0, + "57": 2795345408.0, + "58": 2795345408.0, + "59": 2795345408.0, + "60": 2795345408.0, + "61": 2795345408.0, + "62": 2795345408.0, + "63": 2795345408.0, + "64": 2795345408.0, + "65": 2795345408.0, + "66": 2795345408.0, + "67": 2795345408.0, + "68": 2795345408.0, + "69": 2795345408.0, + "70": 2795345408.0, + "71": 2795345408.0, + "72": 2795345408.0, + "73": 2795345408.0, + "74": 2795345408.0, + "75": 2795345408.0, + "76": 2795345408.0, + "77": 2795345408.0, + "78": 2795345408.0, + "79": 2795345408.0, + "80": 2795345408.0, + "81": 2795345408.0, + "82": 2795345408.0, + "83": 2795345408.0, + "84": 2795345408.0, + "85": 2795345408.0, + "86": 2795345408.0, + "87": 2795345408.0, + "88": 2795345408.0, + "89": 2795345408.0, + "90": 2795345408.0, + "91": 2795345408.0, + "92": 2795345408.0, + "93": 2795345408.0, + "94": 2795345408.0, + "95": 2795345408.0, + "96": 2795345408.0, + "97": 2795345408.0, + "98": 2795345408.0, + "99": 2795345408.0, + "100": 2795345408.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.76623, + "2": 0.41645, + "3": 0.15729, + "4": 0.15655, + "5": 0.15439, + "6": 0.15381, + "7": 0.15343, + "8": 0.15339, + "9": 0.15322, + "10": 0.15355, + "11": 0.15239, + "12": 0.15196, + "13": 0.15235, + "14": 0.15283, + "15": 0.15313, + "16": 0.15425, + "17": 0.15411, + "18": 0.15593, + "19": 0.1533, + "20": 0.15547, + "21": 0.15314, + "22": 0.15551, + "23": 0.1542, + "24": 0.15396, + "25": 0.15334, + "26": 0.1528, + "27": 0.15224, + "28": 0.15377, + "29": 0.15378, + "30": 0.15296, + "31": 0.15277, + "32": 0.15329, + "33": 0.15349, + "34": 0.15291, + "35": 0.15336, + "36": 0.15204, + "37": 0.15238, + "38": 0.15344, + "39": 0.15283, + "40": 0.15272, + "41": 0.15285, + "42": 0.15614, + "43": 0.15406, + "44": 0.15389, + "45": 0.15591, + "46": 0.15294, + "47": 0.15419, + "48": 0.15358, + "49": 0.15525, + "50": 0.15464, + "51": 0.15896, + "52": 0.15708, + "53": 0.15757, + "54": 0.15758, + "55": 0.15641, + "56": 0.15661, + "57": 0.15804, + "58": 0.16037, + "59": 0.15723, + "60": 0.1577, + "61": 0.1593, + "62": 0.15586, + "63": 0.15828, + "64": 0.15836, + "65": 0.16023, + "66": 0.15618, + "67": 0.15583, + "68": 0.15756, + "69": 0.18281, + "70": 0.16635, + "71": 0.19055, + "72": 0.15756, + "73": 0.15973, + "74": 0.16035, + "75": 0.15764, + "76": 0.15689, + "77": 0.15696, + "78": 0.15869, + "79": 0.15855, + "80": 0.15884, + "81": 0.15812, + "82": 0.15771, + "83": 0.15837, + "84": 0.15744, + "85": 0.15771, + "86": 0.15721, + "87": 0.15692, + "88": 0.15759, + "89": 0.15908, + "90": 0.15803, + "91": 0.15786, + "92": 0.15817, + "93": 0.15819, + "94": 0.15883, + "95": 0.15933, + "96": 0.15749, + "97": 0.15871, + "98": 0.1577, + "99": 0.15695, + "100": 0.15802 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json index 3b1e19e8a67..4771e4e3c8c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.81184, "5": 10.85464, "10": 10.88256, "15": 10.80679, "20": 10.63196, "25": 10.47374, "30": 10.39285, "35": 9.96791, "40": 10.1, "45": 9.68344, "50": 9.92465, "55": 9.98132, "60": 9.3523, "65": 9.70213, "70": 9.73809, "75": 8.95616, "80": 9.31501, "85": 8.97886, "90": 9.55812, "95": 9.13529, "100": 9.21091}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1125.0, "5": 1255.0, "10": 1367.0, "15": 1127.0, "20": 1082.0, "25": 1114.0, "30": 1558.0, "35": 1292.0, "40": 1433.0, "45": 1564.0, "50": 1749.0, "55": 1718.0, "60": 1872.0, "65": 2464.0, "70": 2564.0, "75": 1884.0, "80": 1924.0, "85": 2150.0, "90": 2319.0, "95": 2518.0, "100": 2071.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 733859840.0, "5": 733859840.0, "10": 733859840.0, "15": 733859840.0, "20": 733859840.0, "25": 733859840.0, "30": 733859840.0, "35": 733859840.0, "40": 733859840.0, "45": 733859840.0, "50": 733859840.0, "55": 733859840.0, "60": 733859840.0, "65": 733859840.0, "70": 733859840.0, "75": 733859840.0, "80": 733859840.0, "85": 733859840.0, "90": 733859840.0, "95": 733859840.0, "100": 733859840.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 4312984064.0, "5": 4596792832.0, "10": 4596792832.0, "15": 4596792832.0, "20": 4596792832.0, "25": 4596792832.0, "30": 4596792832.0, "35": 4596792832.0, "40": 4596792832.0, "45": 4596792832.0, "50": 4596792832.0, "55": 4596792832.0, "60": 4596792832.0, "65": 4596792832.0, "70": 4596792832.0, "75": 4596792832.0, "80": 4596792832.0, "85": 4596792832.0, "90": 4596792832.0, "95": 4596792832.0, "100": 4596792832.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 17.99007, "5": 0.37673, "10": 0.36645, "15": 0.3643, "20": 0.36375, "25": 0.3652, "30": 0.36495, "35": 0.36435, "40": 0.36456, "45": 0.3636, "50": 0.36345, "55": 0.36651, "60": 0.36683, "65": 0.36434, "70": 0.36726, "75": 0.36517, "80": 0.56121, "85": 0.36207, "90": 0.35913, "95": 0.36112, "100": 0.36099}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80831, + "2": 10.81091, + "3": 10.82881, + "4": 10.82888, + "5": 10.8518, + "6": 10.84129, + "7": 10.839, + "8": 10.80345, + "9": 10.87875, + "10": 10.88064, + "11": 10.87605, + "12": 10.82786, + "13": 10.84179, + "14": 10.81686, + "15": 10.80637, + "16": 10.79763, + "17": 10.76624, + "18": 10.78069, + "19": 10.75188, + "20": 10.63371, + "21": 10.68119, + "22": 10.63819, + "23": 10.75544, + "24": 10.61129, + "25": 10.47478, + "26": 10.59702, + "27": 10.53977, + "28": 10.45162, + "29": 10.39421, + "30": 10.39348, + "31": 10.49397, + "32": 10.32158, + "33": 10.27676, + "34": 10.44629, + "35": 9.96856, + "36": 10.11335, + "37": 10.02306, + "38": 10.37579, + "39": 9.78845, + "40": 10.10124, + "41": 10.12323, + "42": 10.02969, + "43": 10.19869, + "44": 10.05528, + "45": 9.68289, + "46": 9.98143, + "47": 9.92628, + "48": 9.66986, + "49": 9.9198, + "50": 9.9226, + "51": 9.79344, + "52": 9.32777, + "53": 9.65056, + "54": 9.8608, + "55": 9.98167, + "56": 9.81701, + "57": 9.74358, + "58": 9.83, + "59": 9.32869, + "60": 9.35336, + "61": 9.45108, + "62": 10.19054, + "63": 9.35643, + "64": 9.62865, + "65": 9.70227, + "66": 9.52515, + "67": 9.6623, + "68": 9.58775, + "69": 9.38554, + "70": 9.73858, + "71": 9.87574, + "72": 9.69164, + "73": 9.39216, + "74": 9.43911, + "75": 8.95557, + "76": 9.56378, + "77": 9.6144, + "78": 9.39189, + "79": 9.52909, + "80": 9.31414, + "81": 9.70167, + "82": 9.90384, + "83": 9.31614, + "84": 9.47152, + "85": 8.97772, + "86": 9.66417, + "87": 9.43151, + "88": 9.58677, + "89": 9.52297, + "90": 9.55832, + "91": 9.62794, + "92": 9.14014, + "93": 9.42404, + "94": 9.54555, + "95": 9.13499, + "96": 8.75216, + "97": 9.58103, + "98": 9.79018, + "99": 9.37904, + "100": 9.21212 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1004.0, + "2": 1243.0, + "3": 1236.0, + "4": 1235.0, + "5": 1179.0, + "6": 1319.0, + "7": 1173.0, + "8": 1020.0, + "9": 1205.0, + "10": 1270.0, + "11": 1163.0, + "12": 1276.0, + "13": 1398.0, + "14": 1096.0, + "15": 1148.0, + "16": 1104.0, + "17": 1141.0, + "18": 1312.0, + "19": 1200.0, + "20": 1094.0, + "21": 1197.0, + "22": 1286.0, + "23": 1353.0, + "24": 1321.0, + "25": 1053.0, + "26": 1219.0, + "27": 1300.0, + "28": 1428.0, + "29": 1300.0, + "30": 1550.0, + "31": 1500.0, + "32": 1450.0, + "33": 1597.0, + "34": 1487.0, + "35": 1196.0, + "36": 1333.0, + "37": 1457.0, + "38": 1618.0, + "39": 1321.0, + "40": 1467.0, + "41": 1743.0, + "42": 1583.0, + "43": 1726.0, + "44": 1687.0, + "45": 1607.0, + "46": 1887.0, + "47": 1694.0, + "48": 1589.0, + "49": 1754.0, + "50": 1681.0, + "51": 1792.0, + "52": 1765.0, + "53": 1874.0, + "54": 1854.0, + "55": 1816.0, + "56": 1894.0, + "57": 1855.0, + "58": 1733.0, + "59": 1425.0, + "60": 1916.0, + "61": 2236.0, + "62": 2138.0, + "63": 2016.0, + "64": 2084.0, + "65": 2473.0, + "66": 2130.0, + "67": 2275.0, + "68": 2207.0, + "69": 2199.0, + "70": 2462.0, + "71": 2390.0, + "72": 2501.0, + "73": 1957.0, + "74": 2120.0, + "75": 2095.0, + "76": 2325.0, + "77": 2395.0, + "78": 2502.0, + "79": 2700.0, + "80": 1996.0, + "81": 2404.0, + "82": 2501.0, + "83": 2493.0, + "84": 2074.0, + "85": 2167.0, + "86": 2273.0, + "87": 2540.0, + "88": 2295.0, + "89": 2583.0, + "90": 2139.0, + "91": 2390.0, + "92": 1929.0, + "93": 2179.0, + "94": 2522.0, + "95": 2468.0, + "96": 2219.0, + "97": 2145.0, + "98": 2343.0, + "99": 2330.0, + "100": 1971.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2513633792.0, + "2": 2795345408.0, + "3": 2795345408.0, + "4": 2795345408.0, + "5": 2795345408.0, + "6": 2795345408.0, + "7": 2795345408.0, + "8": 2795345408.0, + "9": 2795345408.0, + "10": 2795345408.0, + "11": 2795345408.0, + "12": 2795345408.0, + "13": 2795345408.0, + "14": 2795345408.0, + "15": 2795345408.0, + "16": 2795345408.0, + "17": 2795345408.0, + "18": 2795345408.0, + "19": 2795345408.0, + "20": 2795345408.0, + "21": 2795345408.0, + "22": 2795345408.0, + "23": 2795345408.0, + "24": 2795345408.0, + "25": 2795345408.0, + "26": 2795345408.0, + "27": 2795345408.0, + "28": 2795345408.0, + "29": 2795345408.0, + "30": 2795345408.0, + "31": 2795345408.0, + "32": 2795345408.0, + "33": 2795345408.0, + "34": 2795345408.0, + "35": 2795345408.0, + "36": 2795345408.0, + "37": 2795345408.0, + "38": 2795345408.0, + "39": 2795345408.0, + "40": 2795345408.0, + "41": 2795345408.0, + "42": 2795345408.0, + "43": 2795345408.0, + "44": 2795345408.0, + "45": 2795345408.0, + "46": 2795345408.0, + "47": 2795345408.0, + "48": 2795345408.0, + "49": 2795345408.0, + "50": 2795345408.0, + "51": 2795345408.0, + "52": 2795345408.0, + "53": 2795345408.0, + "54": 2795345408.0, + "55": 2795345408.0, + "56": 2795345408.0, + "57": 2795345408.0, + "58": 2795345408.0, + "59": 2795345408.0, + "60": 2795345408.0, + "61": 2795345408.0, + "62": 2795345408.0, + "63": 2795345408.0, + "64": 2795345408.0, + "65": 2795345408.0, + "66": 2795345408.0, + "67": 2795345408.0, + "68": 2795345408.0, + "69": 2795345408.0, + "70": 2795345408.0, + "71": 2795345408.0, + "72": 2795345408.0, + "73": 2795345408.0, + "74": 2795345408.0, + "75": 2795345408.0, + "76": 2795345408.0, + "77": 2795345408.0, + "78": 2795345408.0, + "79": 2795345408.0, + "80": 2795345408.0, + "81": 2795345408.0, + "82": 2795345408.0, + "83": 2795345408.0, + "84": 2795345408.0, + "85": 2795345408.0, + "86": 2795345408.0, + "87": 2795345408.0, + "88": 2795345408.0, + "89": 2795345408.0, + "90": 2795345408.0, + "91": 2795345408.0, + "92": 2795345408.0, + "93": 2795345408.0, + "94": 2795345408.0, + "95": 2795345408.0, + "96": 2795345408.0, + "97": 2795345408.0, + "98": 2795345408.0, + "99": 2795345408.0, + "100": 2795345408.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.66407, + "2": 0.18828, + "3": 0.15715, + "4": 0.15685, + "5": 0.1544, + "6": 0.15356, + "7": 0.15196, + "8": 0.15101, + "9": 0.15114, + "10": 0.15067, + "11": 0.15113, + "12": 0.15109, + "13": 0.15255, + "14": 0.15181, + "15": 0.15165, + "16": 0.14989, + "17": 0.15094, + "18": 0.15062, + "19": 0.15148, + "20": 0.15014, + "21": 0.15114, + "22": 0.14973, + "23": 0.15192, + "24": 0.15003, + "25": 0.15228, + "26": 0.15066, + "27": 0.15209, + "28": 0.15056, + "29": 0.1516, + "30": 0.15083, + "31": 0.15211, + "32": 0.15028, + "33": 0.1518, + "34": 0.1494, + "35": 0.1521, + "36": 0.15002, + "37": 0.15257, + "38": 0.15095, + "39": 0.1517, + "40": 0.1501, + "41": 0.15352, + "42": 0.15453, + "43": 0.15187, + "44": 0.15281, + "45": 0.15294, + "46": 0.15214, + "47": 0.15376, + "48": 0.15363, + "49": 0.15977, + "50": 0.15249, + "51": 0.15543, + "52": 0.15363, + "53": 0.15379, + "54": 0.15555, + "55": 0.15252, + "56": 0.15295, + "57": 0.15496, + "58": 0.15756, + "59": 0.15345, + "60": 0.15784, + "61": 0.1581, + "62": 0.15302, + "63": 0.15579, + "64": 0.1536, + "65": 0.15523, + "66": 0.15593, + "67": 0.15868, + "68": 0.15303, + "69": 0.1554, + "70": 0.15409, + "71": 0.15229, + "72": 0.15299, + "73": 0.15495, + "74": 0.15601, + "75": 0.15285, + "76": 0.15774, + "77": 0.15171, + "78": 0.15423, + "79": 0.15398, + "80": 0.15445, + "81": 0.15381, + "82": 0.15311, + "83": 0.15584, + "84": 0.15556, + "85": 0.15506, + "86": 0.15314, + "87": 0.15269, + "88": 0.15515, + "89": 0.15923, + "90": 0.15325, + "91": 0.15755, + "92": 0.1543, + "93": 0.15481, + "94": 0.15321, + "95": 0.15397, + "96": 0.15322, + "97": 0.15471, + "98": 0.15631, + "99": 0.15271, + "100": 0.15653 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..cf9f4245915 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80779, + "2": 10.81028, + "3": 10.82881, + "4": 10.82931, + "5": 10.85227, + "6": 10.84112, + "7": 10.83921, + "8": 10.80433, + "9": 10.87802, + "10": 10.88031, + "11": 10.87647, + "12": 10.82798, + "13": 10.84122, + "14": 10.81649, + "15": 10.80572, + "16": 10.7979, + "17": 10.76665, + "18": 10.78004, + "19": 10.7521, + "20": 10.63378, + "21": 10.68086, + "22": 10.63834, + "23": 10.75546, + "24": 10.61158, + "25": 10.47402, + "26": 10.59699, + "27": 10.53929, + "28": 10.45199, + "29": 10.39324, + "30": 10.39328, + "31": 10.49337, + "32": 10.32183, + "33": 10.2767, + "34": 10.44619, + "35": 9.96854, + "36": 10.11277, + "37": 10.02313, + "38": 10.37525, + "39": 9.78827, + "40": 10.10093, + "41": 10.1232, + "42": 10.02925, + "43": 10.19868, + "44": 10.05532, + "45": 9.68289, + "46": 9.98153, + "47": 9.92617, + "48": 9.66965, + "49": 9.91955, + "50": 9.92258 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1058.0, + "2": 1243.0, + "3": 1255.0, + "4": 1229.0, + "5": 1178.0, + "6": 1278.0, + "7": 1129.0, + "8": 955.0, + "9": 1299.0, + "10": 1314.0, + "11": 1220.0, + "12": 1334.0, + "13": 1364.0, + "14": 1153.0, + "15": 1214.0, + "16": 1150.0, + "17": 1176.0, + "18": 1304.0, + "19": 1090.0, + "20": 1170.0, + "21": 1286.0, + "22": 1325.0, + "23": 1353.0, + "24": 1341.0, + "25": 1100.0, + "26": 1174.0, + "27": 1357.0, + "28": 1293.0, + "29": 1259.0, + "30": 1539.0, + "31": 1485.0, + "32": 1422.0, + "33": 1410.0, + "34": 1525.0, + "35": 1163.0, + "36": 1230.0, + "37": 1392.0, + "38": 1570.0, + "39": 1260.0, + "40": 1481.0, + "41": 1687.0, + "42": 1606.0, + "43": 1696.0, + "44": 1648.0, + "45": 1620.0, + "46": 1901.0, + "47": 1627.0, + "48": 1595.0, + "49": 1750.0, + "50": 1723.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4311542784.0, + "2": 4593254400.0, + "3": 4593254400.0, + "4": 4593254400.0, + "5": 4593254400.0, + "6": 4593254400.0, + "7": 4593254400.0, + "8": 4593254400.0, + "9": 4593254400.0, + "10": 4593254400.0, + "11": 4593254400.0, + "12": 4593254400.0, + "13": 4593254400.0, + "14": 4593254400.0, + "15": 4593254400.0, + "16": 4593254400.0, + "17": 4593254400.0, + "18": 4593254400.0, + "19": 4593254400.0, + "20": 4593254400.0, + "21": 4593254400.0, + "22": 4593254400.0, + "23": 4593254400.0, + "24": 4593254400.0, + "25": 4593254400.0, + "26": 4593254400.0, + "27": 4593254400.0, + "28": 4593254400.0, + "29": 4593254400.0, + "30": 4593254400.0, + "31": 4593254400.0, + "32": 4593254400.0, + "33": 4593254400.0, + "34": 4593254400.0, + "35": 4593254400.0, + "36": 4593254400.0, + "37": 4593254400.0, + "38": 4593254400.0, + "39": 4593254400.0, + "40": 4593254400.0, + "41": 4593254400.0, + "42": 4593254400.0, + "43": 4593254400.0, + "44": 4593254400.0, + "45": 4593254400.0, + "46": 4593254400.0, + "47": 4593254400.0, + "48": 4593254400.0, + "49": 4593254400.0, + "50": 4593254400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.15288, + "2": 0.50843, + "3": 0.16717, + "4": 0.16415, + "5": 0.16752, + "6": 0.16666, + "7": 0.16995, + "8": 0.17105, + "9": 0.17027, + "10": 0.17005, + "11": 0.17, + "12": 0.16945, + "13": 0.1682, + "14": 0.1687, + "15": 0.1682, + "16": 0.16899, + "17": 0.16877, + "18": 0.16877, + "19": 0.1688, + "20": 0.16832, + "21": 0.1693, + "22": 0.16953, + "23": 0.16772, + "24": 0.16919, + "25": 0.169, + "26": 0.16905, + "27": 0.16801, + "28": 0.16843, + "29": 0.1685, + "30": 0.16864, + "31": 0.16799, + "32": 0.16279, + "33": 0.16411, + "34": 0.16258, + "35": 0.16352, + "36": 0.16277, + "37": 0.16443, + "38": 0.16253, + "39": 0.16392, + "40": 0.16225, + "41": 0.16316, + "42": 0.16241, + "43": 0.16358, + "44": 0.16211, + "45": 0.16324, + "46": 0.16385, + "47": 0.16427, + "48": 0.16287, + "49": 0.16404, + "50": 0.164 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json index 0a4708ee24f..16019e9879e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.81184, "5": 10.85464, "10": 10.88256, "15": 10.80679, "20": 10.63196, "25": 10.47374, "30": 10.39285, "35": 9.96791, "40": 10.1, "45": 9.68344, "50": 9.92465}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1125.0, "5": 1255.0, "10": 1367.0, "15": 1127.0, "20": 1082.0, "25": 1114.0, "30": 1558.0, "35": 1292.0, "40": 1433.0, "45": 1564.0, "50": 1749.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 733859840.0, "5": 733859840.0, "10": 733859840.0, "15": 733859840.0, "20": 733859840.0, "25": 733859840.0, "30": 733859840.0, "35": 733859840.0, "40": 733859840.0, "45": 733859840.0, "50": 733859840.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 4312984064.0, "5": 4596792832.0, "10": 4596792832.0, "15": 4596792832.0, "20": 4596792832.0, "25": 4596792832.0, "30": 4596792832.0, "35": 4596792832.0, "40": 4596792832.0, "45": 4596792832.0, "50": 4596792832.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 15.07476, "5": 0.37209, "10": 0.36754, "15": 0.36929, "20": 0.36944, "25": 0.36826, "30": 0.3678, "35": 0.36826, "40": 0.36559, "45": 0.36731, "50": 0.36188}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80779, + "2": 10.81028, + "3": 10.82881, + "4": 10.82931, + "5": 10.85227, + "6": 10.84112, + "7": 10.83921, + "8": 10.80433, + "9": 10.87802, + "10": 10.88031, + "11": 10.87647, + "12": 10.82798, + "13": 10.84122, + "14": 10.81649, + "15": 10.80572, + "16": 10.7979, + "17": 10.76665, + "18": 10.78004, + "19": 10.7521, + "20": 10.63378, + "21": 10.68086, + "22": 10.63834, + "23": 10.75546, + "24": 10.61158, + "25": 10.47402, + "26": 10.59699, + "27": 10.53929, + "28": 10.45199, + "29": 10.39324, + "30": 10.39328, + "31": 10.49337, + "32": 10.32183, + "33": 10.2767, + "34": 10.44619, + "35": 9.96854, + "36": 10.11277, + "37": 10.02313, + "38": 10.37525, + "39": 9.78827, + "40": 10.10093, + "41": 10.1232, + "42": 10.02925, + "43": 10.19868, + "44": 10.05532, + "45": 9.68289, + "46": 9.98153, + "47": 9.92617, + "48": 9.66965, + "49": 9.91955, + "50": 9.92258 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1058.0, + "2": 1243.0, + "3": 1255.0, + "4": 1229.0, + "5": 1178.0, + "6": 1278.0, + "7": 1129.0, + "8": 955.0, + "9": 1299.0, + "10": 1314.0, + "11": 1220.0, + "12": 1334.0, + "13": 1364.0, + "14": 1153.0, + "15": 1214.0, + "16": 1150.0, + "17": 1176.0, + "18": 1304.0, + "19": 1090.0, + "20": 1170.0, + "21": 1286.0, + "22": 1325.0, + "23": 1353.0, + "24": 1341.0, + "25": 1100.0, + "26": 1174.0, + "27": 1357.0, + "28": 1293.0, + "29": 1259.0, + "30": 1539.0, + "31": 1485.0, + "32": 1422.0, + "33": 1410.0, + "34": 1525.0, + "35": 1163.0, + "36": 1230.0, + "37": 1392.0, + "38": 1570.0, + "39": 1260.0, + "40": 1481.0, + "41": 1687.0, + "42": 1606.0, + "43": 1696.0, + "44": 1648.0, + "45": 1620.0, + "46": 1901.0, + "47": 1627.0, + "48": 1595.0, + "49": 1750.0, + "50": 1723.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4311542784.0, + "2": 4593254400.0, + "3": 4593254400.0, + "4": 4593254400.0, + "5": 4593254400.0, + "6": 4593254400.0, + "7": 4593254400.0, + "8": 4593254400.0, + "9": 4593254400.0, + "10": 4593254400.0, + "11": 4593254400.0, + "12": 4593254400.0, + "13": 4593254400.0, + "14": 4593254400.0, + "15": 4593254400.0, + "16": 4593254400.0, + "17": 4593254400.0, + "18": 4593254400.0, + "19": 4593254400.0, + "20": 4593254400.0, + "21": 4593254400.0, + "22": 4593254400.0, + "23": 4593254400.0, + "24": 4593254400.0, + "25": 4593254400.0, + "26": 4593254400.0, + "27": 4593254400.0, + "28": 4593254400.0, + "29": 4593254400.0, + "30": 4593254400.0, + "31": 4593254400.0, + "32": 4593254400.0, + "33": 4593254400.0, + "34": 4593254400.0, + "35": 4593254400.0, + "36": 4593254400.0, + "37": 4593254400.0, + "38": 4593254400.0, + "39": 4593254400.0, + "40": 4593254400.0, + "41": 4593254400.0, + "42": 4593254400.0, + "43": 4593254400.0, + "44": 4593254400.0, + "45": 4593254400.0, + "46": 4593254400.0, + "47": 4593254400.0, + "48": 4593254400.0, + "49": 4593254400.0, + "50": 4593254400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.03263, + "2": 0.21266, + "3": 0.17373, + "4": 0.17827, + "5": 0.17392, + "6": 0.17641, + "7": 0.17509, + "8": 0.17211, + "9": 0.17464, + "10": 0.21373, + "11": 0.17143, + "12": 0.17137, + "13": 0.17701, + "14": 0.17242, + "15": 0.16945, + "16": 0.1686, + "17": 0.16945, + "18": 0.16793, + "19": 0.16997, + "20": 0.16992, + "21": 0.17016, + "22": 0.16832, + "23": 0.16853, + "24": 0.16912, + "25": 0.16822, + "26": 0.16908, + "27": 0.16609, + "28": 0.239, + "29": 0.16968, + "30": 0.16763, + "31": 0.16962, + "32": 0.16788, + "33": 0.1681, + "34": 0.16749, + "35": 0.16866, + "36": 0.1697, + "37": 0.16838, + "38": 0.16867, + "39": 0.16699, + "40": 0.17098, + "41": 0.1671, + "42": 0.17036, + "43": 0.16755, + "44": 0.16699, + "45": 0.1678, + "46": 0.17136, + "47": 0.16725, + "48": 0.17257, + "49": 0.16903, + "50": 0.1687 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json new file mode 100644 index 00000000000..fa4447e6f40 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.97215, + "2": 10.96419, + "3": 10.96484, + "4": 10.9625, + "5": 10.96069, + "6": 10.96314, + "7": 10.95542, + "8": 10.96012, + "9": 10.9642, + "10": 10.94938, + "11": 10.95571, + "12": 10.94758, + "13": 10.94696, + "14": 10.95407, + "15": 10.91083, + "16": 10.90945, + "17": 10.89001, + "18": 10.89837, + "19": 10.89189, + "20": 10.80856, + "21": 10.78598, + "22": 10.69678, + "23": 10.79518, + "24": 10.6983, + "25": 10.66637, + "26": 10.71173, + "27": 10.68383, + "28": 10.62344, + "29": 10.65067, + "30": 10.45487, + "31": 10.22475, + "32": 10.52171, + "33": 10.52706, + "34": 10.29181, + "35": 10.33574, + "36": 10.2871, + "37": 10.39557, + "38": 10.2536, + "39": 10.44971, + "40": 10.14791, + "41": 10.19255, + "42": 10.24716, + "43": 9.89482, + "44": 10.00764, + "45": 9.88792, + "46": 9.86452, + "47": 10.16578, + "48": 9.87929, + "49": 9.5693, + "50": 9.95204 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727298.0, + "2": 22924540.0, + "3": 22596662.0, + "4": 23219864.0, + "5": 22714604.0, + "6": 23021138.0, + "7": 22770420.0, + "8": 22926748.0, + "9": 22842456.0, + "10": 22918536.0, + "11": 22500950.0, + "12": 22459666.0, + "13": 22917172.0, + "14": 22388760.0, + "15": 22821456.0, + "16": 22831856.0, + "17": 22818966.0, + "18": 22582904.0, + "19": 22618308.0, + "20": 22695018.0, + "21": 22739412.0, + "22": 22799506.0, + "23": 22539964.0, + "24": 22771406.0, + "25": 22818944.0, + "26": 22548888.0, + "27": 22467554.0, + "28": 22453820.0, + "29": 22529468.0, + "30": 22631974.0, + "31": 22954914.0, + "32": 22584176.0, + "33": 22557850.0, + "34": 22835592.0, + "35": 22787628.0, + "36": 22589872.0, + "37": 22497168.0, + "38": 22896192.0, + "39": 22801888.0, + "40": 22658038.0, + "41": 22659420.0, + "42": 22666532.0, + "43": 22976852.0, + "44": 22746072.0, + "45": 22675348.0, + "46": 22884600.0, + "47": 22633528.0, + "48": 22929392.0, + "49": 22728496.0, + "50": 22905024.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 659815936.0, + "2": 659815936.0, + "3": 659815936.0, + "4": 659815936.0, + "5": 659815936.0, + "6": 659815936.0, + "7": 659815936.0, + "8": 659815936.0, + "9": 659815936.0, + "10": 659815936.0, + "11": 659815936.0, + "12": 659815936.0, + "13": 659815936.0, + "14": 659815936.0, + "15": 659815936.0, + "16": 659815936.0, + "17": 659815936.0, + "18": 659815936.0, + "19": 659815936.0, + "20": 659815936.0, + "21": 659815936.0, + "22": 659815936.0, + "23": 659815936.0, + "24": 659815936.0, + "25": 659815936.0, + "26": 659815936.0, + "27": 659815936.0, + "28": 659815936.0, + "29": 659815936.0, + "30": 659815936.0, + "31": 659815936.0, + "32": 659815936.0, + "33": 659815936.0, + "34": 659815936.0, + "35": 659815936.0, + "36": 659815936.0, + "37": 659815936.0, + "38": 659815936.0, + "39": 659815936.0, + "40": 659815936.0, + "41": 659815936.0, + "42": 659815936.0, + "43": 659815936.0, + "44": 659815936.0, + "45": 659815936.0, + "46": 659815936.0, + "47": 659815936.0, + "48": 659815936.0, + "49": 659815936.0, + "50": 659815936.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2128664064.0, + "2": 2387253760.0, + "3": 2387253760.0, + "4": 2387253760.0, + "5": 2387253760.0, + "6": 2387253760.0, + "7": 2387253760.0, + "8": 2387253760.0, + "9": 2387253760.0, + "10": 2387253760.0, + "11": 2387253760.0, + "12": 2387253760.0, + "13": 2387253760.0, + "14": 2387253760.0, + "15": 2387253760.0, + "16": 2387253760.0, + "17": 2387253760.0, + "18": 2387253760.0, + "19": 2387253760.0, + "20": 2387253760.0, + "21": 2387253760.0, + "22": 2387253760.0, + "23": 2387253760.0, + "24": 2387253760.0, + "25": 2387253760.0, + "26": 2387253760.0, + "27": 2387253760.0, + "28": 2387253760.0, + "29": 2387253760.0, + "30": 2387253760.0, + "31": 2387253760.0, + "32": 2387253760.0, + "33": 2387253760.0, + "34": 2387253760.0, + "35": 2387253760.0, + "36": 2387253760.0, + "37": 2387253760.0, + "38": 2387253760.0, + "39": 2387253760.0, + "40": 2387253760.0, + "41": 2387253760.0, + "42": 2387253760.0, + "43": 2387253760.0, + "44": 2387253760.0, + "45": 2387253760.0, + "46": 2387253760.0, + "47": 2387253760.0, + "48": 2387253760.0, + "49": 2387253760.0, + "50": 2387253760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.87306, + "2": 0.21422, + "3": 0.25104, + "4": 0.16762, + "5": 0.16515, + "6": 0.16439, + "7": 0.16456, + "8": 0.16608, + "9": 0.16475, + "10": 0.16561, + "11": 0.16487, + "12": 0.16315, + "13": 0.16552, + "14": 0.16522, + "15": 0.16481, + "16": 0.16526, + "17": 0.16423, + "18": 0.16401, + "19": 0.16489, + "20": 0.16413, + "21": 0.16376, + "22": 0.16471, + "23": 0.16552, + "24": 0.16391, + "25": 0.15853, + "26": 0.15805, + "27": 0.15784, + "28": 0.15821, + "29": 0.15801, + "30": 0.15776, + "31": 0.15957, + "32": 0.16205, + "33": 0.16126, + "34": 0.15977, + "35": 0.16204, + "36": 0.15979, + "37": 0.16009, + "38": 0.15888, + "39": 0.16022, + "40": 0.1597, + "41": 0.15982, + "42": 0.1593, + "43": 0.15759, + "44": 0.15811, + "45": 0.1573, + "46": 0.15807, + "47": 0.15605, + "48": 0.15694, + "49": 0.15675, + "50": 0.15757 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json index 0cd9c8700a3..775784e5ee0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.9735, - "5": 10.95594, - "10": 10.94989, - "15": 10.9115, - "20": 10.80975, - "25": 10.6662, - "30": 10.45501, - "35": 10.33418, - "40": 10.14646, - "45": 9.89112, - "50": 9.9526 + "1": 10.97215, + "2": 10.96419, + "3": 10.96484, + "4": 10.9625, + "5": 10.96069, + "6": 10.96314, + "7": 10.95542, + "8": 10.96012, + "9": 10.9642, + "10": 10.94938, + "11": 10.95571, + "12": 10.94758, + "13": 10.94696, + "14": 10.95407, + "15": 10.91083, + "16": 10.90945, + "17": 10.89001, + "18": 10.89837, + "19": 10.89189, + "20": 10.80856, + "21": 10.78598, + "22": 10.69678, + "23": 10.79518, + "24": 10.6983, + "25": 10.66637, + "26": 10.71173, + "27": 10.68383, + "28": 10.62344, + "29": 10.65067, + "30": 10.45487, + "31": 10.22475, + "32": 10.52171, + "33": 10.52706, + "34": 10.29181, + "35": 10.33574, + "36": 10.2871, + "37": 10.39557, + "38": 10.2536, + "39": 10.44971, + "40": 10.14791, + "41": 10.19255, + "42": 10.24716, + "43": 9.89482, + "44": 10.00764, + "45": 9.88792, + "46": 9.86452, + "47": 10.16578, + "48": 9.87929, + "49": 9.5693, + "50": 9.95204 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 22727052.0, - "5": 22714228.0, - "10": 22918376.0, - "15": 22820932.0, - "20": 22694228.0, - "25": 22819528.0, - "30": 22631178.0, - "35": 22787526.0, - "40": 22657932.0, - "45": 22674550.0, - "50": 22905400.0 + "1": 22727298.0, + "2": 22924540.0, + "3": 22596662.0, + "4": 23219864.0, + "5": 22714604.0, + "6": 23021138.0, + "7": 22770420.0, + "8": 22926748.0, + "9": 22842456.0, + "10": 22918536.0, + "11": 22500950.0, + "12": 22459666.0, + "13": 22917172.0, + "14": 22388760.0, + "15": 22821456.0, + "16": 22831856.0, + "17": 22818966.0, + "18": 22582904.0, + "19": 22618308.0, + "20": 22695018.0, + "21": 22739412.0, + "22": 22799506.0, + "23": 22539964.0, + "24": 22771406.0, + "25": 22818944.0, + "26": 22548888.0, + "27": 22467554.0, + "28": 22453820.0, + "29": 22529468.0, + "30": 22631974.0, + "31": 22954914.0, + "32": 22584176.0, + "33": 22557850.0, + "34": 22835592.0, + "35": 22787628.0, + "36": 22589872.0, + "37": 22497168.0, + "38": 22896192.0, + "39": 22801888.0, + "40": 22658038.0, + "41": 22659420.0, + "42": 22666532.0, + "43": 22976852.0, + "44": 22746072.0, + "45": 22675348.0, + "46": 22884600.0, + "47": 22633528.0, + "48": 22929392.0, + "49": 22728496.0, + "50": 22905024.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 657718272.0, - "5": 657718272.0, - "10": 657718272.0, - "15": 657718272.0, - "20": 657718272.0, - "25": 657718272.0, - "30": 657718272.0, - "35": 657718272.0, - "40": 657718272.0, - "45": 657718272.0, - "50": 657718272.0 + "1": 659815936.0, + "2": 659815936.0, + "3": 659815936.0, + "4": 659815936.0, + "5": 659815936.0, + "6": 659815936.0, + "7": 659815936.0, + "8": 659815936.0, + "9": 659815936.0, + "10": 659815936.0, + "11": 659815936.0, + "12": 659815936.0, + "13": 659815936.0, + "14": 659815936.0, + "15": 659815936.0, + "16": 659815936.0, + "17": 659815936.0, + "18": 659815936.0, + "19": 659815936.0, + "20": 659815936.0, + "21": 659815936.0, + "22": 659815936.0, + "23": 659815936.0, + "24": 659815936.0, + "25": 659815936.0, + "26": 659815936.0, + "27": 659815936.0, + "28": 659815936.0, + "29": 659815936.0, + "30": 659815936.0, + "31": 659815936.0, + "32": 659815936.0, + "33": 659815936.0, + "34": 659815936.0, + "35": 659815936.0, + "36": 659815936.0, + "37": 659815936.0, + "38": 659815936.0, + "39": 659815936.0, + "40": 659815936.0, + "41": 659815936.0, + "42": 659815936.0, + "43": 659815936.0, + "44": 659815936.0, + "45": 659815936.0, + "46": 659815936.0, + "47": 659815936.0, + "48": 659815936.0, + "49": 659815936.0, + "50": 659815936.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 2129712128.0, - "5": 2385156096.0, - "10": 2385156096.0, - "15": 2385156096.0, - "20": 2385156096.0, - "25": 2385156096.0, - "30": 2385156096.0, - "35": 2385156096.0, - "40": 2385156096.0, - "45": 2385156096.0, - "50": 2385156096.0 + "1": 2128664064.0, + "2": 2387253760.0, + "3": 2387253760.0, + "4": 2387253760.0, + "5": 2387253760.0, + "6": 2387253760.0, + "7": 2387253760.0, + "8": 2387253760.0, + "9": 2387253760.0, + "10": 2387253760.0, + "11": 2387253760.0, + "12": 2387253760.0, + "13": 2387253760.0, + "14": 2387253760.0, + "15": 2387253760.0, + "16": 2387253760.0, + "17": 2387253760.0, + "18": 2387253760.0, + "19": 2387253760.0, + "20": 2387253760.0, + "21": 2387253760.0, + "22": 2387253760.0, + "23": 2387253760.0, + "24": 2387253760.0, + "25": 2387253760.0, + "26": 2387253760.0, + "27": 2387253760.0, + "28": 2387253760.0, + "29": 2387253760.0, + "30": 2387253760.0, + "31": 2387253760.0, + "32": 2387253760.0, + "33": 2387253760.0, + "34": 2387253760.0, + "35": 2387253760.0, + "36": 2387253760.0, + "37": 2387253760.0, + "38": 2387253760.0, + "39": 2387253760.0, + "40": 2387253760.0, + "41": 2387253760.0, + "42": 2387253760.0, + "43": 2387253760.0, + "44": 2387253760.0, + "45": 2387253760.0, + "46": 2387253760.0, + "47": 2387253760.0, + "48": 2387253760.0, + "49": 2387253760.0, + "50": 2387253760.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 20.28193, - "5": 0.17568, - "10": 0.17422, - "15": 0.17474, - "20": 0.17247, - "25": 0.16917, - "30": 0.16924, - "35": 0.16826, - "40": 0.16896, - "45": 0.1662, - "50": 0.16732 + "1": 10.48419, + "2": 0.19482, + "3": 0.26748, + "4": 0.1633, + "5": 0.15828, + "6": 0.15656, + "7": 0.1572, + "8": 0.15759, + "9": 0.15735, + "10": 0.15751, + "11": 0.15648, + "12": 0.15605, + "13": 0.15693, + "14": 0.15672, + "15": 0.15676, + "16": 0.15664, + "17": 0.15683, + "18": 0.15646, + "19": 0.15696, + "20": 0.15623, + "21": 0.15652, + "22": 0.15759, + "23": 0.15729, + "24": 0.15687, + "25": 0.15563, + "26": 0.1575, + "27": 0.15616, + "28": 0.15855, + "29": 0.15771, + "30": 0.15851, + "31": 0.1579, + "32": 0.1587, + "33": 0.1577, + "34": 0.15827, + "35": 0.15808, + "36": 0.15825, + "37": 0.1583, + "38": 0.15836, + "39": 0.15797, + "40": 0.15829, + "41": 0.15787, + "42": 0.15789, + "43": 0.15839, + "44": 0.15862, + "45": 0.15727, + "46": 0.15919, + "47": 0.15859, + "48": 0.15898, + "49": 0.15832, + "50": 0.1586 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json new file mode 100644 index 00000000000..fc938769241 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84562, + "2": 10.8536, + "3": 10.85978, + "4": 10.84186, + "5": 10.8756, + "6": 10.88172, + "7": 10.86719, + "8": 10.84992, + "9": 10.86792, + "10": 10.83289, + "11": 10.91083, + "12": 10.87501, + "13": 10.86463, + "14": 10.8928, + "15": 10.84145, + "16": 10.84108, + "17": 10.82014, + "18": 10.85327, + "19": 10.8576, + "20": 10.80111, + "21": 10.78705, + "22": 10.72832, + "23": 10.812, + "24": 10.74372, + "25": 10.712, + "26": 10.76908, + "27": 10.78641, + "28": 10.73219, + "29": 10.75717, + "30": 10.58162, + "31": 10.43196, + "32": 10.68369, + "33": 10.66821, + "34": 10.49987, + "35": 10.5319, + "36": 10.52076, + "37": 10.59704, + "38": 10.45745, + "39": 10.62078, + "40": 10.35651, + "41": 10.40224, + "42": 10.45518, + "43": 10.11645, + "44": 10.24176, + "45": 10.1377, + "46": 10.11459, + "47": 10.39867, + "48": 10.14178, + "49": 9.8915, + "50": 10.20004 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727180.0, + "2": 22925490.0, + "3": 22595358.0, + "4": 23219094.0, + "5": 22713274.0, + "6": 23020840.0, + "7": 22770150.0, + "8": 22925912.0, + "9": 22842304.0, + "10": 22917686.0, + "11": 22499934.0, + "12": 22458902.0, + "13": 22916544.0, + "14": 22388412.0, + "15": 22820828.0, + "16": 22829442.0, + "17": 22818424.0, + "18": 22582572.0, + "19": 22616516.0, + "20": 22693786.0, + "21": 22738788.0, + "22": 22800098.0, + "23": 22538252.0, + "24": 22771304.0, + "25": 22818714.0, + "26": 22547732.0, + "27": 22467548.0, + "28": 22452916.0, + "29": 22528584.0, + "30": 22630192.0, + "31": 22954950.0, + "32": 22584400.0, + "33": 22557234.0, + "34": 22834292.0, + "35": 22786428.0, + "36": 22588576.0, + "37": 22496998.0, + "38": 22895112.0, + "39": 22800900.0, + "40": 22657380.0, + "41": 22658838.0, + "42": 22666328.0, + "43": 22975596.0, + "44": 22745924.0, + "45": 22674268.0, + "46": 22884128.0, + "47": 22632352.0, + "48": 22927496.0, + "49": 22727204.0, + "50": 22903716.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 611461632.0, + "2": 611461632.0, + "3": 611461632.0, + "4": 611461632.0, + "5": 611461632.0, + "6": 611461632.0, + "7": 611461632.0, + "8": 611461632.0, + "9": 611461632.0, + "10": 611461632.0, + "11": 611461632.0, + "12": 611461632.0, + "13": 611461632.0, + "14": 611461632.0, + "15": 611461632.0, + "16": 611461632.0, + "17": 611461632.0, + "18": 611461632.0, + "19": 611461632.0, + "20": 611461632.0, + "21": 611461632.0, + "22": 611461632.0, + "23": 611461632.0, + "24": 611461632.0, + "25": 611461632.0, + "26": 611461632.0, + "27": 611461632.0, + "28": 611461632.0, + "29": 611461632.0, + "30": 611461632.0, + "31": 611461632.0, + "32": 611461632.0, + "33": 611461632.0, + "34": 611461632.0, + "35": 611461632.0, + "36": 611461632.0, + "37": 611461632.0, + "38": 611461632.0, + "39": 611461632.0, + "40": 611461632.0, + "41": 611461632.0, + "42": 611461632.0, + "43": 611461632.0, + "44": 611461632.0, + "45": 611461632.0, + "46": 611461632.0, + "47": 611461632.0, + "48": 611461632.0, + "49": 611461632.0, + "50": 611461632.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2580665856.0, + "2": 2812987904.0, + "3": 2812987904.0, + "4": 2812987904.0, + "5": 2812987904.0, + "6": 2812987904.0, + "7": 2812987904.0, + "8": 2812987904.0, + "9": 2812987904.0, + "10": 2812987904.0, + "11": 2812987904.0, + "12": 2812987904.0, + "13": 2812987904.0, + "14": 2812987904.0, + "15": 2812987904.0, + "16": 2812987904.0, + "17": 2812987904.0, + "18": 2812987904.0, + "19": 2812987904.0, + "20": 2812987904.0, + "21": 2812987904.0, + "22": 2812987904.0, + "23": 2812987904.0, + "24": 2812987904.0, + "25": 2812987904.0, + "26": 2812987904.0, + "27": 2812987904.0, + "28": 2812987904.0, + "29": 2812987904.0, + "30": 2812987904.0, + "31": 2812987904.0, + "32": 2812987904.0, + "33": 2812987904.0, + "34": 2812987904.0, + "35": 2812987904.0, + "36": 2812987904.0, + "37": 2812987904.0, + "38": 2812987904.0, + "39": 2812987904.0, + "40": 2812987904.0, + "41": 2812987904.0, + "42": 2812987904.0, + "43": 2812987904.0, + "44": 2812987904.0, + "45": 2812987904.0, + "46": 2812987904.0, + "47": 2812987904.0, + "48": 2812987904.0, + "49": 2812987904.0, + "50": 2812987904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.97654, + "2": 0.12833, + "3": 0.09335, + "4": 0.09288, + "5": 0.09223, + "6": 0.09324, + "7": 0.09538, + "8": 0.09184, + "9": 0.09231, + "10": 0.09289, + "11": 0.09189, + "12": 0.09166, + "13": 0.09092, + "14": 0.09055, + "15": 0.09032, + "16": 0.09021, + "17": 0.09123, + "18": 0.09029, + "19": 0.09028, + "20": 0.09042, + "21": 0.08964, + "22": 0.08984, + "23": 0.08956, + "24": 0.08945, + "25": 0.09004, + "26": 0.0892, + "27": 0.08926, + "28": 0.08893, + "29": 0.08891, + "30": 0.08938, + "31": 0.08988, + "32": 0.08972, + "33": 0.08927, + "34": 0.09022, + "35": 0.0899, + "36": 0.0891, + "37": 0.08893, + "38": 0.08927, + "39": 0.08909, + "40": 0.08953, + "41": 0.08971, + "42": 0.08902, + "43": 0.09024, + "44": 0.08936, + "45": 0.0894, + "46": 0.08968, + "47": 0.08906, + "48": 0.08956, + "49": 0.08937, + "50": 0.08941 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json index 117df660805..44d53d6e9d6 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.84528, "5": 10.87387, "10": 10.83535, "15": 10.84228, "20": 10.80167, "25": 10.71149, "30": 10.58467, "35": 10.53287, "40": 10.35652, "45": 10.13839, "50": 10.20005}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 22726236.0, "5": 22714030.0, "10": 22918036.0, "15": 22821520.0, "20": 22693656.0, "25": 22819138.0, "30": 22630790.0, "35": 22787508.0, "40": 22657590.0, "45": 22674508.0, "50": 22904058.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 611461120.0, "5": 611461120.0, "10": 611461120.0, "15": 611461120.0, "20": 611461120.0, "25": 611461120.0, "30": 611461120.0, "35": 611461120.0, "40": 611461120.0, "45": 611461120.0, "50": 611461120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2582211584.0, "5": 2814533632.0, "10": 2814533632.0, "15": 2814533632.0, "20": 2814533632.0, "25": 2814533632.0, "30": 2814533632.0, "35": 2814533632.0, "40": 2814533632.0, "45": 2814533632.0, "50": 2814533632.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 14.65275, "5": 0.11877, "10": 0.11712, "15": 0.11705, "20": 0.11603, "25": 0.11576, "30": 0.11652, "35": 0.11656, "40": 0.11712, "45": 0.11593, "50": 0.11726}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84562, + "2": 10.8536, + "3": 10.85978, + "4": 10.84186, + "5": 10.8756, + "6": 10.88172, + "7": 10.86719, + "8": 10.84992, + "9": 10.86792, + "10": 10.83289, + "11": 10.91083, + "12": 10.87501, + "13": 10.86463, + "14": 10.8928, + "15": 10.84145, + "16": 10.84108, + "17": 10.82014, + "18": 10.85327, + "19": 10.8576, + "20": 10.80111, + "21": 10.78705, + "22": 10.72832, + "23": 10.812, + "24": 10.74372, + "25": 10.712, + "26": 10.76908, + "27": 10.78641, + "28": 10.73219, + "29": 10.75717, + "30": 10.58162, + "31": 10.43196, + "32": 10.68369, + "33": 10.66821, + "34": 10.49987, + "35": 10.5319, + "36": 10.52076, + "37": 10.59704, + "38": 10.45745, + "39": 10.62078, + "40": 10.35651, + "41": 10.40224, + "42": 10.45518, + "43": 10.11645, + "44": 10.24176, + "45": 10.1377, + "46": 10.11459, + "47": 10.39867, + "48": 10.14178, + "49": 9.8915, + "50": 10.20004 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727180.0, + "2": 22925490.0, + "3": 22595358.0, + "4": 23219094.0, + "5": 22713274.0, + "6": 23020840.0, + "7": 22770150.0, + "8": 22925912.0, + "9": 22842304.0, + "10": 22917686.0, + "11": 22499934.0, + "12": 22458902.0, + "13": 22916544.0, + "14": 22388412.0, + "15": 22820828.0, + "16": 22829442.0, + "17": 22818424.0, + "18": 22582572.0, + "19": 22616516.0, + "20": 22693786.0, + "21": 22738788.0, + "22": 22800098.0, + "23": 22538252.0, + "24": 22771304.0, + "25": 22818714.0, + "26": 22547732.0, + "27": 22467548.0, + "28": 22452916.0, + "29": 22528584.0, + "30": 22630192.0, + "31": 22954950.0, + "32": 22584400.0, + "33": 22557234.0, + "34": 22834292.0, + "35": 22786428.0, + "36": 22588576.0, + "37": 22496998.0, + "38": 22895112.0, + "39": 22800900.0, + "40": 22657380.0, + "41": 22658838.0, + "42": 22666328.0, + "43": 22975596.0, + "44": 22745924.0, + "45": 22674268.0, + "46": 22884128.0, + "47": 22632352.0, + "48": 22927496.0, + "49": 22727204.0, + "50": 22903716.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 609364480.0, + "2": 609364480.0, + "3": 609364480.0, + "4": 609364480.0, + "5": 609364480.0, + "6": 609364480.0, + "7": 609364480.0, + "8": 609364480.0, + "9": 609364480.0, + "10": 609364480.0, + "11": 609364480.0, + "12": 609364480.0, + "13": 609364480.0, + "14": 609364480.0, + "15": 609364480.0, + "16": 609364480.0, + "17": 609364480.0, + "18": 609364480.0, + "19": 609364480.0, + "20": 609364480.0, + "21": 609364480.0, + "22": 609364480.0, + "23": 609364480.0, + "24": 609364480.0, + "25": 609364480.0, + "26": 609364480.0, + "27": 609364480.0, + "28": 609364480.0, + "29": 609364480.0, + "30": 609364480.0, + "31": 609364480.0, + "32": 609364480.0, + "33": 609364480.0, + "34": 609364480.0, + "35": 609364480.0, + "36": 609364480.0, + "37": 609364480.0, + "38": 609364480.0, + "39": 609364480.0, + "40": 609364480.0, + "41": 609364480.0, + "42": 609364480.0, + "43": 609364480.0, + "44": 609364480.0, + "45": 609364480.0, + "46": 609364480.0, + "47": 609364480.0, + "48": 609364480.0, + "49": 609364480.0, + "50": 609364480.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2580665856.0, + "2": 2810890752.0, + "3": 2811808256.0, + "4": 2811808256.0, + "5": 2811808256.0, + "6": 2811808256.0, + "7": 2811808256.0, + "8": 2811808256.0, + "9": 2811808256.0, + "10": 2811808256.0, + "11": 2811808256.0, + "12": 2811808256.0, + "13": 2811808256.0, + "14": 2811808256.0, + "15": 2811808256.0, + "16": 2811808256.0, + "17": 2811808256.0, + "18": 2811808256.0, + "19": 2811808256.0, + "20": 2811808256.0, + "21": 2811808256.0, + "22": 2811808256.0, + "23": 2811808256.0, + "24": 2811808256.0, + "25": 2811808256.0, + "26": 2811808256.0, + "27": 2811808256.0, + "28": 2811808256.0, + "29": 2811808256.0, + "30": 2811808256.0, + "31": 2811808256.0, + "32": 2811808256.0, + "33": 2811808256.0, + "34": 2811808256.0, + "35": 2811808256.0, + "36": 2811808256.0, + "37": 2811808256.0, + "38": 2811808256.0, + "39": 2811808256.0, + "40": 2811808256.0, + "41": 2811808256.0, + "42": 2811808256.0, + "43": 2811808256.0, + "44": 2811808256.0, + "45": 2811808256.0, + "46": 2811808256.0, + "47": 2811808256.0, + "48": 2811808256.0, + "49": 2811808256.0, + "50": 2811808256.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.118, + "2": 0.12375, + "3": 0.31133, + "4": 0.09209, + "5": 0.09124, + "6": 0.09155, + "7": 0.09163, + "8": 0.0915, + "9": 0.09161, + "10": 0.09407, + "11": 0.09038, + "12": 0.09031, + "13": 0.09069, + "14": 0.09024, + "15": 0.09043, + "16": 0.08996, + "17": 0.09133, + "18": 0.09072, + "19": 0.09048, + "20": 0.09016, + "21": 0.09061, + "22": 0.09073, + "23": 0.09098, + "24": 0.09135, + "25": 0.09235, + "26": 0.09059, + "27": 0.09009, + "28": 0.09049, + "29": 0.09147, + "30": 0.09097, + "31": 0.09098, + "32": 0.09045, + "33": 0.09082, + "34": 0.08994, + "35": 0.09054, + "36": 0.09124, + "37": 0.09063, + "38": 0.08989, + "39": 0.09234, + "40": 0.09165, + "41": 0.09179, + "42": 0.09165, + "43": 0.09235, + "44": 0.09147, + "45": 0.0922, + "46": 0.09192, + "47": 0.09138, + "48": 0.09278, + "49": 0.09145, + "50": 0.09175 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json new file mode 100644 index 00000000000..57fe2806d3f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88765, + "2": 10.90324, + "3": 10.87803, + "4": 10.87967, + "5": 10.91013, + "6": 10.91377, + "7": 10.89285, + "8": 10.88408, + "9": 10.89749, + "10": 10.87762, + "11": 10.88984, + "12": 10.89502, + "13": 10.90738, + "14": 10.90831, + "15": 10.86285, + "16": 10.85944, + "17": 10.8221, + "18": 10.84343, + "19": 10.83663, + "20": 10.74694, + "21": 10.73044, + "22": 10.62623, + "23": 10.74379, + "24": 10.63229, + "25": 10.59789, + "26": 10.64585, + "27": 10.64828, + "28": 10.59539, + "29": 10.60615, + "30": 10.40161, + "31": 10.18116, + "32": 10.49112, + "33": 10.48508, + "34": 10.25199, + "35": 10.30784, + "36": 10.25604, + "37": 10.36576, + "38": 10.21673, + "39": 10.40989, + "40": 10.11047, + "41": 10.15883, + "42": 10.22753, + "43": 9.8745, + "44": 9.97803, + "45": 9.8591, + "46": 9.8546, + "47": 10.15264, + "48": 9.86489, + "49": 9.5555, + "50": 9.92088, + "51": 9.86094, + "52": 9.75697, + "53": 10.07633, + "54": 9.96082, + "55": 9.88565, + "56": 9.6349, + "57": 9.4925, + "58": 9.83099, + "59": 9.59122, + "60": 9.50798, + "61": 9.7061, + "62": 9.98413, + "63": 9.37604, + "64": 9.77938, + "65": 8.95852, + "66": 9.70596, + "67": 9.37402, + "68": 9.78683, + "69": 9.78932, + "70": 9.72766, + "71": 9.61135, + "72": 9.59178, + "73": 9.49896, + "74": 8.95742, + "75": 9.42469, + "76": 9.09651, + "77": 10.06653, + "78": 9.73149, + "79": 9.37959, + "80": 9.40394, + "81": 9.48277, + "82": 9.69318, + "83": 9.31104, + "84": 9.4139, + "85": 9.61469, + "86": 9.07793, + "87": 9.59662, + "88": 9.74827, + "89": 9.60196, + "90": 9.81239, + "91": 9.34524, + "92": 9.36524, + "93": 9.07745, + "94": 8.83182, + "95": 9.521, + "96": 9.52525, + "97": 9.31322, + "98": 9.677, + "99": 8.88904, + "100": 9.40063 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1753.0, + "2": 1863.0, + "3": 1781.0, + "4": 1850.0, + "5": 2021.0, + "6": 1987.0, + "7": 2006.0, + "8": 1657.0, + "9": 1961.0, + "10": 1491.0, + "11": 2010.0, + "12": 1846.0, + "13": 1934.0, + "14": 1954.0, + "15": 1942.0, + "16": 1969.0, + "17": 1936.0, + "18": 1836.0, + "19": 1841.0, + "20": 1684.0, + "21": 1958.0, + "22": 1782.0, + "23": 2112.0, + "24": 1651.0, + "25": 1713.0, + "26": 1878.0, + "27": 1837.0, + "28": 2025.0, + "29": 2115.0, + "30": 1867.0, + "31": 1622.0, + "32": 1939.0, + "33": 2167.0, + "34": 1867.0, + "35": 2035.0, + "36": 2086.0, + "37": 2477.0, + "38": 2296.0, + "39": 2505.0, + "40": 2150.0, + "41": 2315.0, + "42": 2239.0, + "43": 1941.0, + "44": 2169.0, + "45": 2172.0, + "46": 2231.0, + "47": 2604.0, + "48": 2429.0, + "49": 2289.0, + "50": 2529.0, + "51": 2742.0, + "52": 2671.0, + "53": 3066.0, + "54": 2782.0, + "55": 2510.0, + "56": 2874.0, + "57": 2304.0, + "58": 3111.0, + "59": 2862.0, + "60": 2374.0, + "61": 2977.0, + "62": 2740.0, + "63": 2394.0, + "64": 3232.0, + "65": 2720.0, + "66": 3277.0, + "67": 2810.0, + "68": 2830.0, + "69": 3094.0, + "70": 3327.0, + "71": 3106.0, + "72": 2261.0, + "73": 3147.0, + "74": 1902.0, + "75": 2545.0, + "76": 2905.0, + "77": 3468.0, + "78": 3432.0, + "79": 3336.0, + "80": 3434.0, + "81": 3605.0, + "82": 3269.0, + "83": 2891.0, + "84": 3343.0, + "85": 3501.0, + "86": 2786.0, + "87": 3872.0, + "88": 3019.0, + "89": 3407.0, + "90": 3023.0, + "91": 2630.0, + "92": 3186.0, + "93": 2746.0, + "94": 3526.0, + "95": 3414.0, + "96": 3546.0, + "97": 3339.0, + "98": 3758.0, + "99": 3058.0, + "100": 3454.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 462388736.0, + "2": 462388736.0, + "3": 462388736.0, + "4": 462388736.0, + "5": 462388736.0, + "6": 462388736.0, + "7": 462388736.0, + "8": 462388736.0, + "9": 462388736.0, + "10": 462388736.0, + "11": 462388736.0, + "12": 462388736.0, + "13": 462388736.0, + "14": 462388736.0, + "15": 462388736.0, + "16": 462388736.0, + "17": 462388736.0, + "18": 462388736.0, + "19": 462388736.0, + "20": 462388736.0, + "21": 462388736.0, + "22": 462388736.0, + "23": 462388736.0, + "24": 462388736.0, + "25": 462388736.0, + "26": 462388736.0, + "27": 462388736.0, + "28": 462388736.0, + "29": 462388736.0, + "30": 462388736.0, + "31": 462388736.0, + "32": 462388736.0, + "33": 462388736.0, + "34": 462388736.0, + "35": 462388736.0, + "36": 462388736.0, + "37": 462388736.0, + "38": 462388736.0, + "39": 462388736.0, + "40": 462388736.0, + "41": 462388736.0, + "42": 462388736.0, + "43": 462388736.0, + "44": 462388736.0, + "45": 462388736.0, + "46": 462388736.0, + "47": 462388736.0, + "48": 462388736.0, + "49": 462388736.0, + "50": 462388736.0, + "51": 462388736.0, + "52": 462388736.0, + "53": 462388736.0, + "54": 462388736.0, + "55": 462388736.0, + "56": 462388736.0, + "57": 462388736.0, + "58": 462388736.0, + "59": 462388736.0, + "60": 462388736.0, + "61": 462388736.0, + "62": 462388736.0, + "63": 462388736.0, + "64": 462388736.0, + "65": 462388736.0, + "66": 462388736.0, + "67": 462388736.0, + "68": 462388736.0, + "69": 462388736.0, + "70": 462388736.0, + "71": 462388736.0, + "72": 462388736.0, + "73": 462388736.0, + "74": 462388736.0, + "75": 462388736.0, + "76": 462388736.0, + "77": 462388736.0, + "78": 462388736.0, + "79": 462388736.0, + "80": 462388736.0, + "81": 462388736.0, + "82": 462388736.0, + "83": 462388736.0, + "84": 462388736.0, + "85": 462388736.0, + "86": 462388736.0, + "87": 462388736.0, + "88": 462388736.0, + "89": 462388736.0, + "90": 462388736.0, + "91": 462388736.0, + "92": 462388736.0, + "93": 462388736.0, + "94": 462388736.0, + "95": 462388736.0, + "96": 462388736.0, + "97": 462388736.0, + "98": 462388736.0, + "99": 462388736.0, + "100": 462388736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1549081088.0, + "2": 1677508096.0, + "3": 1677508096.0, + "4": 1677508096.0, + "5": 1677508096.0, + "6": 1677508096.0, + "7": 1677508096.0, + "8": 1677508096.0, + "9": 1677508096.0, + "10": 1677508096.0, + "11": 1677508096.0, + "12": 1677508096.0, + "13": 1677508096.0, + "14": 1677508096.0, + "15": 1677508096.0, + "16": 1677508096.0, + "17": 1677508096.0, + "18": 1677508096.0, + "19": 1677508096.0, + "20": 1677508096.0, + "21": 1677508096.0, + "22": 1677508096.0, + "23": 1677508096.0, + "24": 1677508096.0, + "25": 1677508096.0, + "26": 1677508096.0, + "27": 1677508096.0, + "28": 1677508096.0, + "29": 1677508096.0, + "30": 1677508096.0, + "31": 1677508096.0, + "32": 1677508096.0, + "33": 1677508096.0, + "34": 1677508096.0, + "35": 1677508096.0, + "36": 1677508096.0, + "37": 1677508096.0, + "38": 1677508096.0, + "39": 1677508096.0, + "40": 1677508096.0, + "41": 1677508096.0, + "42": 1677508096.0, + "43": 1677508096.0, + "44": 1677508096.0, + "45": 1677508096.0, + "46": 1677508096.0, + "47": 1677508096.0, + "48": 1677508096.0, + "49": 1677508096.0, + "50": 1677508096.0, + "51": 1677508096.0, + "52": 1677508096.0, + "53": 1677508096.0, + "54": 1677508096.0, + "55": 1677508096.0, + "56": 1677508096.0, + "57": 1677508096.0, + "58": 1677508096.0, + "59": 1677508096.0, + "60": 1677508096.0, + "61": 1677508096.0, + "62": 1677508096.0, + "63": 1677508096.0, + "64": 1677508096.0, + "65": 1677508096.0, + "66": 1677508096.0, + "67": 1677508096.0, + "68": 1677508096.0, + "69": 1677508096.0, + "70": 1677508096.0, + "71": 1677508096.0, + "72": 1677508096.0, + "73": 1677508096.0, + "74": 1677508096.0, + "75": 1677508096.0, + "76": 1677508096.0, + "77": 1677508096.0, + "78": 1677508096.0, + "79": 1677508096.0, + "80": 1677508096.0, + "81": 1677508096.0, + "82": 1677508096.0, + "83": 1677508096.0, + "84": 1677508096.0, + "85": 1677508096.0, + "86": 1677508096.0, + "87": 1677508096.0, + "88": 1677508096.0, + "89": 1677508096.0, + "90": 1677508096.0, + "91": 1677508096.0, + "92": 1677508096.0, + "93": 1677508096.0, + "94": 1677508096.0, + "95": 1677508096.0, + "96": 1677508096.0, + "97": 1677508096.0, + "98": 1677508096.0, + "99": 1677508096.0, + "100": 1677508096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.74038, + "2": 0.18941, + "3": 0.17443, + "4": 0.17279, + "5": 0.17221, + "6": 0.17285, + "7": 0.17288, + "8": 0.17181, + "9": 0.168, + "10": 0.16829, + "11": 0.16793, + "12": 0.16785, + "13": 0.16712, + "14": 0.17584, + "15": 0.16768, + "16": 0.16694, + "17": 0.16606, + "18": 0.16539, + "19": 0.16505, + "20": 0.16548, + "21": 0.16486, + "22": 0.16481, + "23": 0.1694, + "24": 0.16549, + "25": 0.17312, + "26": 0.16555, + "27": 0.17362, + "28": 0.16604, + "29": 0.1708, + "30": 0.16752, + "31": 0.16804, + "32": 0.16714, + "33": 0.16525, + "34": 0.16728, + "35": 0.16428, + "36": 0.17043, + "37": 0.16532, + "38": 0.16525, + "39": 0.16555, + "40": 0.16531, + "41": 0.16563, + "42": 0.16563, + "43": 0.16499, + "44": 0.16632, + "45": 0.16537, + "46": 0.16563, + "47": 0.16529, + "48": 0.1657, + "49": 0.16462, + "50": 0.16554, + "51": 0.17653, + "52": 0.17191, + "53": 0.17217, + "54": 0.17273, + "55": 0.17175, + "56": 0.1728, + "57": 0.17152, + "58": 0.17298, + "59": 0.17049, + "60": 0.16917, + "61": 0.16777, + "62": 0.16726, + "63": 0.16773, + "64": 0.16713, + "65": 0.16836, + "66": 0.1678, + "67": 0.1679, + "68": 0.16772, + "69": 0.16815, + "70": 0.16816, + "71": 0.16728, + "72": 0.16778, + "73": 0.16743, + "74": 0.16783, + "75": 0.16824, + "76": 0.16759, + "77": 0.16828, + "78": 0.16749, + "79": 0.16757, + "80": 0.16781, + "81": 0.16773, + "82": 0.16701, + "83": 0.16756, + "84": 0.16842, + "85": 0.16707, + "86": 0.16752, + "87": 0.16807, + "88": 0.17901, + "89": 0.17301, + "90": 0.1702, + "91": 0.17493, + "92": 0.17105, + "93": 0.16969, + "94": 0.16931, + "95": 0.17031, + "96": 0.16969, + "97": 0.17032, + "98": 0.17059, + "99": 0.17, + "100": 0.17572 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json index 053ded61519..3be93706d81 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.88734, "5": 10.90948, "10": 10.87763, "15": 10.86371, "20": 10.7509, "25": 10.59917, "30": 10.40104, "35": 10.30793, "40": 10.10902, "45": 9.85831, "50": 9.92111, "55": 9.88529, "60": 9.50737, "65": 8.95828, "70": 9.72733, "75": 9.42571, "80": 9.40559, "85": 9.61572, "90": 9.81277, "95": 9.52119, "100": 9.40111}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1614.0, "5": 2021.0, "10": 1399.0, "15": 1872.0, "20": 1692.0, "25": 1784.0, "30": 1900.0, "35": 2081.0, "40": 2233.0, "45": 2144.0, "50": 2513.0, "55": 2493.0, "60": 2440.0, "65": 2763.0, "70": 3219.0, "75": 2601.0, "80": 3341.0, "85": 3433.0, "90": 3019.0, "95": 3417.0, "100": 3372.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 462455808.0, "5": 462455808.0, "10": 462455808.0, "15": 462455808.0, "20": 462455808.0, "25": 462455808.0, "30": 462455808.0, "35": 462455808.0, "40": 462455808.0, "45": 462455808.0, "50": 462455808.0, "55": 462455808.0, "60": 462455808.0, "65": 462455808.0, "70": 462455808.0, "75": 462455808.0, "80": 462455808.0, "85": 462455808.0, "90": 462455808.0, "95": 462455808.0, "100": 462455808.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2529822720.0, "5": 2658249728.0, "10": 2658249728.0, "15": 2658249728.0, "20": 2658249728.0, "25": 2658249728.0, "30": 2658249728.0, "35": 2658249728.0, "40": 2658249728.0, "45": 2658249728.0, "50": 2658249728.0, "55": 2658249728.0, "60": 2658249728.0, "65": 2658249728.0, "70": 2658249728.0, "75": 2658249728.0, "80": 2658249728.0, "85": 2658249728.0, "90": 2658249728.0, "95": 2658249728.0, "100": 2658249728.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 11.76278, "5": 0.18711, "10": 0.18109, "15": 0.18221, "20": 0.18082, "25": 0.18031, "30": 0.18054, "35": 0.17683, "40": 0.17853, "45": 0.17695, "50": 0.17851, "55": 0.1773, "60": 0.17609, "65": 0.18072, "70": 0.18082, "75": 0.17652, "80": 0.17661, "85": 0.18002, "90": 0.17586, "95": 0.17539, "100": 0.17485}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88765, + "2": 10.90324, + "3": 10.87803, + "4": 10.87967, + "5": 10.91013, + "6": 10.91377, + "7": 10.89285, + "8": 10.88408, + "9": 10.89749, + "10": 10.87762, + "11": 10.88984, + "12": 10.89502, + "13": 10.90738, + "14": 10.90831, + "15": 10.86285, + "16": 10.85944, + "17": 10.8221, + "18": 10.84343, + "19": 10.83663, + "20": 10.74694, + "21": 10.73044, + "22": 10.62623, + "23": 10.74379, + "24": 10.63229, + "25": 10.59789, + "26": 10.64585, + "27": 10.64828, + "28": 10.59539, + "29": 10.60615, + "30": 10.40161, + "31": 10.18116, + "32": 10.49112, + "33": 10.48508, + "34": 10.25199, + "35": 10.30784, + "36": 10.25604, + "37": 10.36576, + "38": 10.21673, + "39": 10.40989, + "40": 10.11047, + "41": 10.15883, + "42": 10.22753, + "43": 9.8745, + "44": 9.97803, + "45": 9.8591, + "46": 9.8546, + "47": 10.15264, + "48": 9.86489, + "49": 9.5555, + "50": 9.92088, + "51": 9.86094, + "52": 9.75697, + "53": 10.07633, + "54": 9.96082, + "55": 9.88565, + "56": 9.6349, + "57": 9.4925, + "58": 9.83099, + "59": 9.59122, + "60": 9.50798, + "61": 9.7061, + "62": 9.98413, + "63": 9.37604, + "64": 9.77938, + "65": 8.95852, + "66": 9.70596, + "67": 9.37402, + "68": 9.78683, + "69": 9.78932, + "70": 9.72766, + "71": 9.61135, + "72": 9.59178, + "73": 9.49896, + "74": 8.95742, + "75": 9.42469, + "76": 9.09651, + "77": 10.06653, + "78": 9.73149, + "79": 9.37959, + "80": 9.40394, + "81": 9.48277, + "82": 9.69318, + "83": 9.31104, + "84": 9.4139, + "85": 9.61469, + "86": 9.07793, + "87": 9.59662, + "88": 9.74827, + "89": 9.60196, + "90": 9.81239, + "91": 9.34524, + "92": 9.36524, + "93": 9.07745, + "94": 8.83182, + "95": 9.521, + "96": 9.52525, + "97": 9.31322, + "98": 9.677, + "99": 8.88904, + "100": 9.40063 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1753.0, + "2": 1863.0, + "3": 1781.0, + "4": 1850.0, + "5": 2021.0, + "6": 1987.0, + "7": 2006.0, + "8": 1657.0, + "9": 1961.0, + "10": 1491.0, + "11": 2010.0, + "12": 1846.0, + "13": 1934.0, + "14": 1954.0, + "15": 1942.0, + "16": 1969.0, + "17": 1936.0, + "18": 1836.0, + "19": 1841.0, + "20": 1684.0, + "21": 1958.0, + "22": 1782.0, + "23": 2112.0, + "24": 1651.0, + "25": 1713.0, + "26": 1878.0, + "27": 1837.0, + "28": 2025.0, + "29": 2115.0, + "30": 1867.0, + "31": 1622.0, + "32": 1939.0, + "33": 2167.0, + "34": 1867.0, + "35": 2035.0, + "36": 2086.0, + "37": 2477.0, + "38": 2296.0, + "39": 2505.0, + "40": 2150.0, + "41": 2315.0, + "42": 2239.0, + "43": 1941.0, + "44": 2169.0, + "45": 2172.0, + "46": 2231.0, + "47": 2604.0, + "48": 2429.0, + "49": 2289.0, + "50": 2529.0, + "51": 2742.0, + "52": 2671.0, + "53": 3066.0, + "54": 2782.0, + "55": 2510.0, + "56": 2874.0, + "57": 2304.0, + "58": 3111.0, + "59": 2862.0, + "60": 2374.0, + "61": 2977.0, + "62": 2740.0, + "63": 2394.0, + "64": 3232.0, + "65": 2720.0, + "66": 3277.0, + "67": 2810.0, + "68": 2830.0, + "69": 3094.0, + "70": 3327.0, + "71": 3106.0, + "72": 2261.0, + "73": 3147.0, + "74": 1902.0, + "75": 2545.0, + "76": 2905.0, + "77": 3468.0, + "78": 3432.0, + "79": 3336.0, + "80": 3434.0, + "81": 3605.0, + "82": 3269.0, + "83": 2891.0, + "84": 3343.0, + "85": 3501.0, + "86": 2786.0, + "87": 3872.0, + "88": 3019.0, + "89": 3407.0, + "90": 3023.0, + "91": 2630.0, + "92": 3186.0, + "93": 2746.0, + "94": 3526.0, + "95": 3414.0, + "96": 3546.0, + "97": 3339.0, + "98": 3758.0, + "99": 3058.0, + "100": 3454.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 462388736.0, + "2": 462388736.0, + "3": 462388736.0, + "4": 462388736.0, + "5": 462388736.0, + "6": 462388736.0, + "7": 462388736.0, + "8": 462388736.0, + "9": 462388736.0, + "10": 462388736.0, + "11": 462388736.0, + "12": 462388736.0, + "13": 462388736.0, + "14": 462388736.0, + "15": 462388736.0, + "16": 462388736.0, + "17": 462388736.0, + "18": 462388736.0, + "19": 462388736.0, + "20": 462388736.0, + "21": 462388736.0, + "22": 462388736.0, + "23": 462388736.0, + "24": 462388736.0, + "25": 462388736.0, + "26": 462388736.0, + "27": 462388736.0, + "28": 462388736.0, + "29": 462388736.0, + "30": 462388736.0, + "31": 462388736.0, + "32": 462388736.0, + "33": 462388736.0, + "34": 462388736.0, + "35": 462388736.0, + "36": 462388736.0, + "37": 462388736.0, + "38": 462388736.0, + "39": 462388736.0, + "40": 462388736.0, + "41": 462388736.0, + "42": 462388736.0, + "43": 462388736.0, + "44": 462388736.0, + "45": 462388736.0, + "46": 462388736.0, + "47": 462388736.0, + "48": 462388736.0, + "49": 462388736.0, + "50": 462388736.0, + "51": 462388736.0, + "52": 462388736.0, + "53": 462388736.0, + "54": 462388736.0, + "55": 462388736.0, + "56": 462388736.0, + "57": 462388736.0, + "58": 462388736.0, + "59": 462388736.0, + "60": 462388736.0, + "61": 462388736.0, + "62": 462388736.0, + "63": 462388736.0, + "64": 462388736.0, + "65": 462388736.0, + "66": 462388736.0, + "67": 462388736.0, + "68": 462388736.0, + "69": 462388736.0, + "70": 462388736.0, + "71": 462388736.0, + "72": 462388736.0, + "73": 462388736.0, + "74": 462388736.0, + "75": 462388736.0, + "76": 462388736.0, + "77": 462388736.0, + "78": 462388736.0, + "79": 462388736.0, + "80": 462388736.0, + "81": 462388736.0, + "82": 462388736.0, + "83": 462388736.0, + "84": 462388736.0, + "85": 462388736.0, + "86": 462388736.0, + "87": 462388736.0, + "88": 462388736.0, + "89": 462388736.0, + "90": 462388736.0, + "91": 462388736.0, + "92": 462388736.0, + "93": 462388736.0, + "94": 462388736.0, + "95": 462388736.0, + "96": 462388736.0, + "97": 462388736.0, + "98": 462388736.0, + "99": 462388736.0, + "100": 462388736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1549081088.0, + "2": 1677508096.0, + "3": 1677508096.0, + "4": 1677508096.0, + "5": 1677508096.0, + "6": 1677508096.0, + "7": 1677508096.0, + "8": 1677508096.0, + "9": 1677508096.0, + "10": 1677508096.0, + "11": 1677508096.0, + "12": 1677508096.0, + "13": 1677508096.0, + "14": 1677508096.0, + "15": 1677508096.0, + "16": 1677508096.0, + "17": 1677508096.0, + "18": 1677508096.0, + "19": 1677508096.0, + "20": 1677508096.0, + "21": 1677508096.0, + "22": 1677508096.0, + "23": 1677508096.0, + "24": 1677508096.0, + "25": 1677508096.0, + "26": 1677508096.0, + "27": 1677508096.0, + "28": 1677508096.0, + "29": 1677508096.0, + "30": 1677508096.0, + "31": 1677508096.0, + "32": 1677508096.0, + "33": 1677508096.0, + "34": 1677508096.0, + "35": 1677508096.0, + "36": 1677508096.0, + "37": 1677508096.0, + "38": 1677508096.0, + "39": 1677508096.0, + "40": 1677508096.0, + "41": 1677508096.0, + "42": 1677508096.0, + "43": 1677508096.0, + "44": 1677508096.0, + "45": 1677508096.0, + "46": 1677508096.0, + "47": 1677508096.0, + "48": 1677508096.0, + "49": 1677508096.0, + "50": 1677508096.0, + "51": 1677508096.0, + "52": 1677508096.0, + "53": 1677508096.0, + "54": 1677508096.0, + "55": 1677508096.0, + "56": 1677508096.0, + "57": 1677508096.0, + "58": 1677508096.0, + "59": 1677508096.0, + "60": 1677508096.0, + "61": 1677508096.0, + "62": 1677508096.0, + "63": 1677508096.0, + "64": 1677508096.0, + "65": 1677508096.0, + "66": 1677508096.0, + "67": 1677508096.0, + "68": 1677508096.0, + "69": 1677508096.0, + "70": 1677508096.0, + "71": 1677508096.0, + "72": 1677508096.0, + "73": 1677508096.0, + "74": 1677508096.0, + "75": 1677508096.0, + "76": 1677508096.0, + "77": 1677508096.0, + "78": 1677508096.0, + "79": 1677508096.0, + "80": 1677508096.0, + "81": 1677508096.0, + "82": 1677508096.0, + "83": 1677508096.0, + "84": 1677508096.0, + "85": 1677508096.0, + "86": 1677508096.0, + "87": 1677508096.0, + "88": 1677508096.0, + "89": 1677508096.0, + "90": 1677508096.0, + "91": 1677508096.0, + "92": 1677508096.0, + "93": 1677508096.0, + "94": 1677508096.0, + "95": 1677508096.0, + "96": 1677508096.0, + "97": 1677508096.0, + "98": 1677508096.0, + "99": 1677508096.0, + "100": 1677508096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.17153, + "2": 0.2103, + "3": 0.21541, + "4": 0.21948, + "5": 0.17282, + "6": 0.16921, + "7": 0.1711, + "8": 0.16967, + "9": 0.17064, + "10": 0.16972, + "11": 0.1696, + "12": 0.1701, + "13": 0.16923, + "14": 0.16942, + "15": 0.16782, + "16": 0.17, + "17": 0.16748, + "18": 0.16821, + "19": 0.16739, + "20": 0.16883, + "21": 0.16894, + "22": 0.16847, + "23": 0.16846, + "24": 0.16887, + "25": 0.16905, + "26": 0.16873, + "27": 0.16876, + "28": 0.16868, + "29": 0.1706, + "30": 0.17379, + "31": 0.17109, + "32": 0.17107, + "33": 0.17072, + "34": 0.17137, + "35": 0.17105, + "36": 0.17106, + "37": 0.17077, + "38": 0.17115, + "39": 0.17067, + "40": 0.17057, + "41": 0.17099, + "42": 0.17074, + "43": 0.17091, + "44": 0.17078, + "45": 0.17104, + "46": 0.17055, + "47": 0.17137, + "48": 0.17086, + "49": 0.17081, + "50": 0.17053, + "51": 0.17448, + "52": 0.16607, + "53": 0.16686, + "54": 0.16608, + "55": 0.16654, + "56": 0.16591, + "57": 0.16614, + "58": 0.1659, + "59": 0.16577, + "60": 0.16589, + "61": 0.16557, + "62": 0.16528, + "63": 0.16612, + "64": 0.1658, + "65": 0.16543, + "66": 0.1651, + "67": 0.16559, + "68": 0.16502, + "69": 0.16533, + "70": 0.16636, + "71": 0.16516, + "72": 0.1657, + "73": 0.1656, + "74": 0.16521, + "75": 0.16623, + "76": 0.16628, + "77": 0.16593, + "78": 0.16615, + "79": 0.1658, + "80": 0.16904, + "81": 0.16665, + "82": 0.16575, + "83": 0.16623, + "84": 0.16603, + "85": 0.16577, + "86": 0.16568, + "87": 0.16525, + "88": 0.16531, + "89": 0.16616, + "90": 0.16544, + "91": 0.16581, + "92": 0.16545, + "93": 0.16603, + "94": 0.16501, + "95": 0.16632, + "96": 0.16545, + "97": 0.16577, + "98": 0.19996, + "99": 0.19154, + "100": 0.19156 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..1cf5408b8ba --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.7942, + "2": 0.22889, + "3": 0.19794, + "4": 0.19876, + "5": 0.19802, + "6": 0.19732, + "7": 0.19829, + "8": 0.19869, + "9": 0.20221, + "10": 0.19735, + "11": 0.1983, + "12": 0.19776, + "13": 0.19772, + "14": 0.19813, + "15": 0.20156, + "16": 0.19951, + "17": 0.19694, + "18": 0.19906, + "19": 0.19681, + "20": 0.19852, + "21": 0.20083, + "22": 0.19703, + "23": 0.19864, + "24": 0.1981, + "25": 0.19771, + "26": 0.19763, + "27": 0.19879, + "28": 0.20245, + "29": 0.19781, + "30": 0.19864, + "31": 0.19901, + "32": 0.19832, + "33": 0.19887, + "34": 0.20272, + "35": 0.19918, + "36": 0.19934, + "37": 0.19832, + "38": 0.19851, + "39": 0.19719, + "40": 0.20496, + "41": 0.19802, + "42": 0.19778, + "43": 0.19799, + "44": 0.19797, + "45": 0.21191, + "46": 0.20272, + "47": 0.20241, + "48": 0.202, + "49": 0.20032, + "50": 0.20198, + "51": 0.21717, + "52": 0.21085, + "53": 0.20963, + "54": 0.20989, + "55": 0.20877, + "56": 0.20983, + "57": 0.21317, + "58": 0.21422, + "59": 0.2112, + "60": 0.21308, + "61": 0.21283, + "62": 0.21356, + "63": 0.21093, + "64": 0.20492, + "65": 0.20132, + "66": 0.20054, + "67": 0.19914, + "68": 0.19918, + "69": 0.19966, + "70": 0.19998, + "71": 0.19998, + "72": 0.19839, + "73": 0.19973, + "74": 0.19856, + "75": 0.19805, + "76": 0.19968, + "77": 0.19757, + "78": 0.19789, + "79": 0.19675, + "80": 0.1975, + "81": 0.19765, + "82": 0.19742, + "83": 0.19785, + "84": 0.19665, + "85": 0.19764, + "86": 0.19704, + "87": 0.19705, + "88": 0.19866, + "89": 0.19749, + "90": 0.1966, + "91": 0.19783, + "92": 0.19848, + "93": 0.19735, + "94": 0.19683, + "95": 0.19659, + "96": 0.19746, + "97": 0.19792, + "98": 0.19802, + "99": 0.19675, + "100": 0.19692 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json index b47c82b17f0..16e4a038563 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.92705, - "5": 10.92795, - "10": 10.90786, - "15": 10.88314, - "20": 10.77629, - "25": 10.59141, - "30": 10.39192, - "35": 10.29686, - "40": 10.0964, - "45": 9.84464, - "50": 9.90918, - "55": 9.87762, - "60": 9.49121, - "65": 8.94236, - "70": 9.72266, - "75": 9.41912, - "80": 9.40077, - "85": 9.61207, - "90": 9.81015, - "95": 9.51716, - "100": 9.4015 + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1627.0, - "5": 2010.0, - "10": 1368.0, - "15": 1897.0, - "20": 1626.0, - "25": 1743.0, - "30": 1930.0, - "35": 1954.0, - "40": 2199.0, - "45": 2068.0, - "50": 2460.0, - "55": 2427.0, - "60": 2380.0, - "65": 2657.0, - "70": 3265.0, - "75": 2675.0, - "80": 3434.0, - "85": 3302.0, - "90": 3230.0, - "95": 3340.0, - "100": 3291.0 + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 487096320.0, - "5": 487096320.0, - "10": 487096320.0, - "15": 487096320.0, - "20": 487096320.0, - "25": 487096320.0, - "30": 487096320.0, - "35": 487096320.0, - "40": 487096320.0, - "45": 487096320.0, - "50": 487096320.0, - "55": 487096320.0, - "60": 487096320.0, - "65": 487096320.0, - "70": 487096320.0, - "75": 487096320.0, - "80": 487096320.0, - "85": 487096320.0, - "90": 487096320.0, - "95": 487096320.0, - "100": 487096320.0 + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1719035904.0, - "5": 1900157952.0, - "10": 1901729792.0, - "15": 1901729792.0, - "20": 1901729792.0, - "25": 1901729792.0, - "30": 1901729792.0, - "35": 1901729792.0, - "40": 1901729792.0, - "45": 1901729792.0, - "50": 1901729792.0, - "55": 1901729792.0, - "60": 1901729792.0, - "65": 1901729792.0, - "70": 1901729792.0, - "75": 1901729792.0, - "80": 1901729792.0, - "85": 1901729792.0, - "90": 1901729792.0, - "95": 1901729792.0, - "100": 1901729792.0 + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 14.4466, - "5": 0.23058, - "10": 0.23299, - "15": 0.23075, - "20": 0.23163, - "25": 0.23303, - "30": 0.22991, - "35": 0.22977, - "40": 0.22747, - "45": 0.22974, - "50": 0.52211, - "55": 0.23173, - "60": 0.23326, - "65": 0.23498, - "70": 0.23013, - "75": 0.22876, - "80": 0.22947, - "85": 0.22706, - "90": 0.22612, - "95": 0.22523, - "100": 0.22404 + "1": 7.63807, + "2": 0.24295, + "3": 0.21281, + "4": 0.20931, + "5": 0.20554, + "6": 0.20827, + "7": 0.20618, + "8": 0.20701, + "9": 0.2077, + "10": 0.20875, + "11": 0.20704, + "12": 0.20735, + "13": 0.20734, + "14": 0.20659, + "15": 0.2071, + "16": 0.20766, + "17": 0.20579, + "18": 0.20511, + "19": 0.20563, + "20": 0.20589, + "21": 0.205, + "22": 0.20541, + "23": 0.2062, + "24": 0.20485, + "25": 0.20487, + "26": 0.20458, + "27": 0.20496, + "28": 0.20545, + "29": 0.20623, + "30": 0.20511, + "31": 0.20822, + "32": 0.20482, + "33": 0.20538, + "34": 0.20452, + "35": 0.21063, + "36": 0.20987, + "37": 0.20831, + "38": 0.2088, + "39": 0.20816, + "40": 0.20875, + "41": 0.20857, + "42": 0.20959, + "43": 0.20886, + "44": 0.2086, + "45": 0.20776, + "46": 0.20831, + "47": 0.20853, + "48": 0.2086, + "49": 0.20813, + "50": 0.209, + "51": 0.20574, + "52": 0.19892, + "53": 0.19904, + "54": 0.19867, + "55": 0.19897, + "56": 0.20031, + "57": 0.19874, + "58": 0.19971, + "59": 0.2002, + "60": 0.19847, + "61": 0.19948, + "62": 0.20017, + "63": 0.19926, + "64": 0.19923, + "65": 0.19974, + "66": 0.19915, + "67": 0.19992, + "68": 0.19949, + "69": 0.19842, + "70": 0.19824, + "71": 0.2012, + "72": 0.20144, + "73": 0.20339, + "74": 0.19815, + "75": 0.19802, + "76": 0.19898, + "77": 0.20003, + "78": 0.20017, + "79": 0.20157, + "80": 0.20266, + "81": 0.20004, + "82": 0.19937, + "83": 0.2008, + "84": 0.2009, + "85": 0.20194, + "86": 0.2015, + "87": 0.20004, + "88": 0.20091, + "89": 0.19998, + "90": 0.19993, + "91": 0.20008, + "92": 0.19991, + "93": 0.19979, + "94": 0.19939, + "95": 0.20098, + "96": 0.20045, + "97": 0.19917, + "98": 0.20012, + "99": 0.19963, + "100": 0.19848 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..4378b779118 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.19385, + "2": 0.24311, + "3": 0.20769, + "4": 0.20606, + "5": 0.21034, + "6": 0.20875, + "7": 0.20511, + "8": 0.2045, + "9": 0.20566, + "10": 0.20495, + "11": 0.20684, + "12": 0.20533, + "13": 0.20573, + "14": 0.20691, + "15": 0.20612, + "16": 0.20502, + "17": 0.20553, + "18": 0.20502, + "19": 0.20618, + "20": 0.20587, + "21": 0.20529, + "22": 0.20461, + "23": 0.20575, + "24": 0.2056, + "25": 0.20434, + "26": 0.20473, + "27": 0.20499, + "28": 0.20543, + "29": 0.20597, + "30": 0.20594, + "31": 0.20485, + "32": 0.20584, + "33": 0.20569, + "34": 0.20368, + "35": 0.20375, + "36": 0.20415, + "37": 0.20296, + "38": 0.20379, + "39": 0.20482, + "40": 0.20372, + "41": 0.20369, + "42": 0.2037, + "43": 0.20368, + "44": 0.2036, + "45": 0.20913, + "46": 0.20299, + "47": 0.20291, + "48": 0.20437, + "49": 0.2033, + "50": 0.20293, + "51": 0.21225, + "52": 0.20266, + "53": 0.20231, + "54": 0.20341, + "55": 0.20168, + "56": 0.20234, + "57": 0.20234, + "58": 0.20311, + "59": 0.20312, + "60": 0.20314, + "61": 0.20332, + "62": 0.20225, + "63": 0.20251, + "64": 0.20258, + "65": 0.20214, + "66": 0.2022, + "67": 0.20405, + "68": 0.20192, + "69": 0.2033, + "70": 0.20207, + "71": 0.20216, + "72": 0.20251, + "73": 0.20231, + "74": 0.20203, + "75": 0.20107, + "76": 0.20252, + "77": 0.20263, + "78": 0.20334, + "79": 0.20378, + "80": 0.20072, + "81": 0.20029, + "82": 0.19967, + "83": 0.20354, + "84": 0.20035, + "85": 0.201, + "86": 0.20061, + "87": 0.20132, + "88": 0.20043, + "89": 0.20079, + "90": 0.20125, + "91": 0.20103, + "92": 0.19988, + "93": 0.20137, + "94": 0.20191, + "95": 0.20085, + "96": 0.20182, + "97": 0.20169, + "98": 0.20353, + "99": 0.2004, + "100": 0.20044 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json index 053168da21b..97ea213f560 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92795, "10": 10.90786, "15": 10.88314, "20": 10.77629, "25": 10.59141, "30": 10.39192, "35": 10.29686, "40": 10.0964, "45": 9.84464, "50": 9.90918, "55": 9.87762, "60": 9.49121, "65": 8.94236, "70": 9.72266, "75": 9.41912, "80": 9.40077, "85": 9.61207, "90": 9.81015, "95": 9.51716, "100": 9.4015}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1627.0, "5": 2010.0, "10": 1368.0, "15": 1897.0, "20": 1626.0, "25": 1743.0, "30": 1930.0, "35": 1954.0, "40": 2199.0, "45": 2068.0, "50": 2460.0, "55": 2427.0, "60": 2380.0, "65": 2657.0, "70": 3265.0, "75": 2675.0, "80": 3434.0, "85": 3302.0, "90": 3230.0, "95": 3340.0, "100": 3291.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 487096320.0, "5": 487096320.0, "10": 487096320.0, "15": 487096320.0, "20": 487096320.0, "25": 487096320.0, "30": 487096320.0, "35": 487096320.0, "40": 487096320.0, "45": 487096320.0, "50": 487096320.0, "55": 487096320.0, "60": 487096320.0, "65": 487096320.0, "70": 487096320.0, "75": 487096320.0, "80": 487096320.0, "85": 487096320.0, "90": 487096320.0, "95": 487096320.0, "100": 487096320.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1720084480.0, "5": 1900157952.0, "10": 1900157952.0, "15": 1900157952.0, "20": 1900157952.0, "25": 1900157952.0, "30": 1900157952.0, "35": 1900157952.0, "40": 1900157952.0, "45": 1900157952.0, "50": 1900157952.0, "55": 1900157952.0, "60": 1900157952.0, "65": 1900157952.0, "70": 1900157952.0, "75": 1900157952.0, "80": 1900157952.0, "85": 1900157952.0, "90": 1900157952.0, "95": 1900157952.0, "100": 1900157952.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.26256, "5": 0.218, "10": 0.21609, "15": 0.21516, "20": 0.2149, "25": 0.21592, "30": 0.21243, "35": 0.21175, "40": 0.21098, "45": 0.21116, "50": 0.21161, "55": 0.21221, "60": 0.21264, "65": 0.21113, "70": 0.21176, "75": 0.22341, "80": 0.22245, "85": 0.22222, "90": 0.22203, "95": 0.21302, "100": 0.21583}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.25138, + "2": 0.23075, + "3": 0.20054, + "4": 0.20395, + "5": 0.20085, + "6": 0.19693, + "7": 0.1984, + "8": 0.19691, + "9": 0.19734, + "10": 0.19831, + "11": 0.19755, + "12": 0.20036, + "13": 0.19718, + "14": 0.20205, + "15": 0.19931, + "16": 0.1974, + "17": 0.19891, + "18": 0.19725, + "19": 0.19744, + "20": 0.19621, + "21": 0.19556, + "22": 0.1957, + "23": 0.19653, + "24": 0.19561, + "25": 0.19465, + "26": 0.19483, + "27": 0.19566, + "28": 0.19514, + "29": 0.19571, + "30": 0.19512, + "31": 0.19603, + "32": 0.19794, + "33": 0.19597, + "34": 0.20052, + "35": 0.19938, + "36": 0.19968, + "37": 0.19971, + "38": 0.19989, + "39": 0.20233, + "40": 0.20594, + "41": 0.20596, + "42": 0.20875, + "43": 0.20692, + "44": 0.20224, + "45": 0.20492, + "46": 0.20483, + "47": 0.20404, + "48": 0.20062, + "49": 0.1998, + "50": 0.19944, + "51": 0.21056, + "52": 0.20322, + "53": 0.20394, + "54": 0.20267, + "55": 0.20305, + "56": 0.20261, + "57": 0.20266, + "58": 0.2023, + "59": 0.20259, + "60": 0.20297, + "61": 0.20333, + "62": 0.20344, + "63": 0.20255, + "64": 0.20203, + "65": 0.20288, + "66": 0.20295, + "67": 0.20276, + "68": 0.20255, + "69": 0.20306, + "70": 0.20225, + "71": 0.20236, + "72": 0.20262, + "73": 0.2033, + "74": 0.20279, + "75": 0.20276, + "76": 0.20185, + "77": 0.20283, + "78": 0.20284, + "79": 0.2021, + "80": 0.20273, + "81": 0.20261, + "82": 0.20101, + "83": 0.20222, + "84": 0.20269, + "85": 0.20272, + "86": 0.20286, + "87": 0.20079, + "88": 0.20309, + "89": 0.2026, + "90": 0.20209, + "91": 0.20371, + "92": 0.20302, + "93": 0.20226, + "94": 0.20222, + "95": 0.20289, + "96": 0.20273, + "97": 0.20346, + "98": 0.20283, + "99": 0.20241, + "100": 0.20343 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json new file mode 100644 index 00000000000..f6faa6eb158 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90497, + "5": 10.92805, + "6": 10.93671, + "7": 10.90403, + "8": 10.92231, + "9": 10.91252, + "10": 10.90851, + "11": 10.89338, + "12": 10.9208, + "13": 10.91494, + "14": 10.92144, + "15": 10.88435, + "16": 10.87458, + "17": 10.83918, + "18": 10.87307, + "19": 10.85328, + "20": 10.77494, + "21": 10.74759, + "22": 10.63148, + "23": 10.75623, + "24": 10.65568, + "25": 10.59221, + "26": 10.65325, + "27": 10.64881, + "28": 10.59654, + "29": 10.61012, + "30": 10.39283, + "31": 10.15719, + "32": 10.49221, + "33": 10.47943, + "34": 10.24015, + "35": 10.29711, + "36": 10.24562, + "37": 10.35281, + "38": 10.20533, + "39": 10.40423, + "40": 10.09553, + "41": 10.15279, + "42": 10.21879, + "43": 9.85526, + "44": 9.96244, + "45": 9.84617, + "46": 9.83798, + "47": 10.1388, + "48": 9.85687, + "49": 9.53746, + "50": 9.90877, + "51": 9.84971, + "52": 9.74156, + "53": 10.06324, + "54": 9.94584, + "55": 9.87735, + "56": 9.62744, + "57": 9.4726, + "58": 9.82907, + "59": 9.58298, + "60": 9.49182, + "61": 9.6996, + "62": 9.98091, + "63": 9.37212, + "64": 9.77558, + "65": 8.94327, + "66": 9.69991, + "67": 9.3641, + "68": 9.78706, + "69": 9.78396, + "70": 9.72291, + "71": 9.60749, + "72": 9.58417, + "73": 9.4909, + "74": 8.94863, + "75": 9.41807, + "76": 9.08721, + "77": 10.06284, + "78": 9.729, + "79": 9.37087, + "80": 9.40029, + "81": 9.47753, + "82": 9.69123, + "83": 9.30764, + "84": 9.4125, + "85": 9.61132, + "86": 9.07624, + "87": 9.59459, + "88": 9.74769, + "89": 9.60678, + "90": 9.81079, + "91": 9.34443, + "92": 9.36534, + "93": 9.07741, + "94": 8.82974, + "95": 9.51676, + "96": 9.52545, + "97": 9.31031, + "98": 9.67811, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 59.0, + "2": 59.0, + "3": 57.0, + "4": 55.0, + "5": 68.0, + "6": 60.0, + "7": 58.0, + "8": 63.0, + "9": 62.0, + "10": 47.0, + "11": 57.0, + "12": 48.0, + "13": 68.0, + "14": 58.0, + "15": 75.0, + "16": 67.0, + "17": 59.0, + "18": 56.0, + "19": 72.0, + "20": 52.0, + "21": 60.0, + "22": 69.0, + "23": 77.0, + "24": 58.0, + "25": 71.0, + "26": 68.0, + "27": 55.0, + "28": 94.0, + "29": 57.0, + "30": 66.0, + "31": 60.0, + "32": 64.0, + "33": 69.0, + "34": 73.0, + "35": 73.0, + "36": 72.0, + "37": 89.0, + "38": 72.0, + "39": 75.0, + "40": 82.0, + "41": 82.0, + "42": 74.0, + "43": 90.0, + "44": 69.0, + "45": 83.0, + "46": 68.0, + "47": 74.0, + "48": 95.0, + "49": 81.0, + "50": 83.0, + "51": 77.0, + "52": 100.0, + "53": 71.0, + "54": 67.0, + "55": 70.0, + "56": 83.0, + "57": 74.0, + "58": 106.0, + "59": 72.0, + "60": 98.0, + "61": 67.0, + "62": 73.0, + "63": 77.0, + "64": 94.0, + "65": 82.0, + "66": 87.0, + "67": 65.0, + "68": 78.0, + "69": 59.0, + "70": 102.0, + "71": 82.0, + "72": 60.0, + "73": 96.0, + "74": 61.0, + "75": 64.0, + "76": 70.0, + "77": 84.0, + "78": 93.0, + "79": 102.0, + "80": 71.0, + "81": 88.0, + "82": 85.0, + "83": 75.0, + "84": 69.0, + "85": 84.0, + "86": 66.0, + "87": 93.0, + "88": 96.0, + "89": 73.0, + "90": 77.0, + "91": 66.0, + "92": 86.0, + "93": 63.0, + "94": 60.0, + "95": 70.0, + "96": 65.0, + "97": 67.0, + "98": 96.0, + "99": 54.0, + "100": 77.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 516063744.0, + "2": 516063744.0, + "3": 516063744.0, + "4": 516063744.0, + "5": 516063744.0, + "6": 516063744.0, + "7": 516063744.0, + "8": 516063744.0, + "9": 516063744.0, + "10": 516063744.0, + "11": 516063744.0, + "12": 516063744.0, + "13": 516063744.0, + "14": 516063744.0, + "15": 516063744.0, + "16": 516063744.0, + "17": 516063744.0, + "18": 516063744.0, + "19": 516063744.0, + "20": 516063744.0, + "21": 516063744.0, + "22": 516063744.0, + "23": 516063744.0, + "24": 516063744.0, + "25": 516063744.0, + "26": 516063744.0, + "27": 516063744.0, + "28": 516063744.0, + "29": 516063744.0, + "30": 516063744.0, + "31": 516063744.0, + "32": 516063744.0, + "33": 516063744.0, + "34": 516063744.0, + "35": 516063744.0, + "36": 516063744.0, + "37": 516063744.0, + "38": 516063744.0, + "39": 516063744.0, + "40": 516063744.0, + "41": 516063744.0, + "42": 516063744.0, + "43": 516063744.0, + "44": 516063744.0, + "45": 516063744.0, + "46": 516063744.0, + "47": 516063744.0, + "48": 516063744.0, + "49": 516063744.0, + "50": 516063744.0, + "51": 516063744.0, + "52": 516063744.0, + "53": 516063744.0, + "54": 516063744.0, + "55": 516063744.0, + "56": 516063744.0, + "57": 516063744.0, + "58": 516063744.0, + "59": 516063744.0, + "60": 516063744.0, + "61": 516063744.0, + "62": 516063744.0, + "63": 516063744.0, + "64": 516063744.0, + "65": 516063744.0, + "66": 516063744.0, + "67": 516063744.0, + "68": 516063744.0, + "69": 516063744.0, + "70": 516063744.0, + "71": 516063744.0, + "72": 516063744.0, + "73": 516063744.0, + "74": 516063744.0, + "75": 516063744.0, + "76": 516063744.0, + "77": 516063744.0, + "78": 516063744.0, + "79": 516063744.0, + "80": 516063744.0, + "81": 516063744.0, + "82": 516063744.0, + "83": 516063744.0, + "84": 516063744.0, + "85": 516063744.0, + "86": 516063744.0, + "87": 516063744.0, + "88": 516063744.0, + "89": 516063744.0, + "90": 516063744.0, + "91": 516063744.0, + "92": 516063744.0, + "93": 516063744.0, + "94": 516063744.0, + "95": 516063744.0, + "96": 516063744.0, + "97": 516063744.0, + "98": 516063744.0, + "99": 516063744.0, + "100": 516063744.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1697019392.0, + "2": 1877092864.0, + "3": 1877092864.0, + "4": 1877092864.0, + "5": 1877092864.0, + "6": 1877092864.0, + "7": 1877092864.0, + "8": 1877092864.0, + "9": 1877092864.0, + "10": 1877092864.0, + "11": 1877092864.0, + "12": 1877092864.0, + "13": 1877092864.0, + "14": 1877092864.0, + "15": 1877092864.0, + "16": 1877092864.0, + "17": 1877092864.0, + "18": 1877092864.0, + "19": 1877092864.0, + "20": 1877092864.0, + "21": 1877092864.0, + "22": 1877092864.0, + "23": 1877092864.0, + "24": 1877092864.0, + "25": 1877092864.0, + "26": 1877092864.0, + "27": 1877092864.0, + "28": 1877092864.0, + "29": 1877092864.0, + "30": 1877092864.0, + "31": 1877092864.0, + "32": 1877092864.0, + "33": 1877092864.0, + "34": 1877092864.0, + "35": 1877092864.0, + "36": 1877092864.0, + "37": 1877092864.0, + "38": 1877092864.0, + "39": 1877092864.0, + "40": 1877092864.0, + "41": 1877092864.0, + "42": 1877092864.0, + "43": 1877092864.0, + "44": 1877092864.0, + "45": 1877092864.0, + "46": 1877092864.0, + "47": 1877092864.0, + "48": 1877092864.0, + "49": 1877092864.0, + "50": 1877092864.0, + "51": 1877092864.0, + "52": 1877092864.0, + "53": 1877092864.0, + "54": 1877092864.0, + "55": 1877092864.0, + "56": 1877092864.0, + "57": 1877092864.0, + "58": 1877092864.0, + "59": 1877092864.0, + "60": 1877092864.0, + "61": 1877092864.0, + "62": 1877092864.0, + "63": 1877092864.0, + "64": 1877092864.0, + "65": 1877092864.0, + "66": 1877092864.0, + "67": 1877092864.0, + "68": 1877092864.0, + "69": 1877092864.0, + "70": 1877092864.0, + "71": 1877092864.0, + "72": 1877092864.0, + "73": 1877092864.0, + "74": 1877092864.0, + "75": 1877092864.0, + "76": 1877092864.0, + "77": 1877092864.0, + "78": 1877092864.0, + "79": 1877092864.0, + "80": 1877092864.0, + "81": 1877092864.0, + "82": 1877092864.0, + "83": 1877092864.0, + "84": 1877092864.0, + "85": 1877092864.0, + "86": 1877092864.0, + "87": 1877092864.0, + "88": 1877092864.0, + "89": 1877092864.0, + "90": 1877092864.0, + "91": 1877092864.0, + "92": 1877092864.0, + "93": 1877092864.0, + "94": 1877092864.0, + "95": 1877092864.0, + "96": 1877092864.0, + "97": 1877092864.0, + "98": 1877092864.0, + "99": 1877092864.0, + "100": 1877092864.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.79688, + "2": 0.23426, + "3": 0.19644, + "4": 0.195, + "5": 0.19474, + "6": 0.19503, + "7": 0.19553, + "8": 0.19608, + "9": 0.1942, + "10": 0.19493, + "11": 0.19477, + "12": 0.19436, + "13": 0.19434, + "14": 0.19471, + "15": 0.19477, + "16": 0.19465, + "17": 0.19394, + "18": 0.19358, + "19": 0.19755, + "20": 0.19602, + "21": 0.19501, + "22": 0.1946, + "23": 0.19563, + "24": 0.19522, + "25": 0.19502, + "26": 0.19664, + "27": 0.19509, + "28": 0.19415, + "29": 0.19426, + "30": 0.19405, + "31": 0.19405, + "32": 0.19558, + "33": 0.19428, + "34": 0.19418, + "35": 0.19493, + "36": 0.19478, + "37": 0.19511, + "38": 0.19481, + "39": 0.19506, + "40": 0.19573, + "41": 0.19576, + "42": 0.19504, + "43": 0.19379, + "44": 0.19366, + "45": 0.19309, + "46": 0.19331, + "47": 0.19424, + "48": 0.21152, + "49": 0.2113, + "50": 0.21175, + "51": 0.20739, + "52": 0.19757, + "53": 0.19742, + "54": 0.19771, + "55": 0.19922, + "56": 0.19903, + "57": 0.19826, + "58": 0.19889, + "59": 0.1988, + "60": 0.19894, + "61": 0.19841, + "62": 0.19853, + "63": 0.19879, + "64": 0.19776, + "65": 0.19833, + "66": 0.19911, + "67": 0.19919, + "68": 0.19768, + "69": 0.19855, + "70": 0.19789, + "71": 0.19851, + "72": 0.19815, + "73": 0.19885, + "74": 0.19845, + "75": 0.19827, + "76": 0.19878, + "77": 0.19962, + "78": 0.19912, + "79": 0.19891, + "80": 0.1985, + "81": 0.19933, + "82": 0.1977, + "83": 0.19774, + "84": 0.19798, + "85": 0.19781, + "86": 0.19763, + "87": 0.1977, + "88": 0.19934, + "89": 0.19771, + "90": 0.19835, + "91": 0.19886, + "92": 0.19768, + "93": 0.19818, + "94": 0.19769, + "95": 0.19836, + "96": 0.19833, + "97": 0.19773, + "98": 0.1988, + "99": 0.1973, + "100": 0.19801 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json index 59a50d2f033..96cf765384a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92799, "10": 10.90789, "15": 10.88313, "20": 10.77626, "25": 10.59138, "30": 10.39195, "35": 10.29687, "40": 10.0964, "45": 9.84466, "50": 9.9092, "55": 9.87764, "60": 9.49121, "65": 8.94232, "70": 9.72263, "75": 9.41912, "80": 9.40079, "85": 9.61209, "90": 9.81012, "95": 9.51718, "100": 9.40153}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 68.0, "5": 64.0, "10": 61.0, "15": 58.0, "20": 64.0, "25": 58.0, "30": 85.0, "35": 66.0, "40": 85.0, "45": 84.0, "50": 85.0, "55": 69.0, "60": 67.0, "65": 65.0, "70": 92.0, "75": 51.0, "80": 98.0, "85": 86.0, "90": 83.0, "95": 72.0, "100": 73.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 488144896.0, "5": 488144896.0, "10": 488144896.0, "15": 488144896.0, "20": 488144896.0, "25": 489193472.0, "30": 488144896.0, "35": 488144896.0, "40": 488144896.0, "45": 488144896.0, "50": 488144896.0, "55": 488144896.0, "60": 488144896.0, "65": 488144896.0, "70": 489193472.0, "75": 489193472.0, "80": 489193472.0, "85": 488144896.0, "90": 488144896.0, "95": 488144896.0, "100": 489193472.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2158389248.0, "5": 2340559872.0, "10": 2340559872.0, "15": 2340559872.0, "20": 2340559872.0, "25": 2340559872.0, "30": 2340559872.0, "35": 2340559872.0, "40": 2340559872.0, "45": 2340559872.0, "50": 2340559872.0, "55": 2340559872.0, "60": 2340559872.0, "65": 2340559872.0, "70": 2340559872.0, "75": 2340559872.0, "80": 2340559872.0, "85": 2340559872.0, "90": 2340559872.0, "95": 2340559872.0, "100": 2340559872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 15.21811, "5": 0.22614, "10": 0.22685, "15": 0.22633, "20": 0.22582, "25": 0.22604, "30": 0.22747, "35": 0.22529, "40": 0.22441, "45": 0.22458, "50": 0.22397, "55": 0.21965, "60": 0.22066, "65": 0.22265, "70": 0.22234, "75": 0.2195, "80": 0.21889, "85": 0.21987, "90": 0.21875, "95": 0.21918, "100": 0.21913}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90497, + "5": 10.92805, + "6": 10.93671, + "7": 10.90403, + "8": 10.92231, + "9": 10.91252, + "10": 10.90851, + "11": 10.89338, + "12": 10.9208, + "13": 10.91494, + "14": 10.92144, + "15": 10.88435, + "16": 10.87458, + "17": 10.83918, + "18": 10.87307, + "19": 10.85328, + "20": 10.77494, + "21": 10.74759, + "22": 10.63148, + "23": 10.75623, + "24": 10.65568, + "25": 10.59221, + "26": 10.65325, + "27": 10.64881, + "28": 10.59654, + "29": 10.61012, + "30": 10.39283, + "31": 10.15719, + "32": 10.49221, + "33": 10.47943, + "34": 10.24015, + "35": 10.29711, + "36": 10.24562, + "37": 10.35281, + "38": 10.20533, + "39": 10.40423, + "40": 10.09553, + "41": 10.15279, + "42": 10.21879, + "43": 9.85526, + "44": 9.96244, + "45": 9.84617, + "46": 9.83798, + "47": 10.1388, + "48": 9.85687, + "49": 9.53746, + "50": 9.90877, + "51": 9.84971, + "52": 9.74156, + "53": 10.06324, + "54": 9.94584, + "55": 9.87735, + "56": 9.62744, + "57": 9.4726, + "58": 9.82907, + "59": 9.58298, + "60": 9.49182, + "61": 9.6996, + "62": 9.98091, + "63": 9.37212, + "64": 9.77558, + "65": 8.94327, + "66": 9.69991, + "67": 9.3641, + "68": 9.78706, + "69": 9.78396, + "70": 9.72291, + "71": 9.60749, + "72": 9.58417, + "73": 9.4909, + "74": 8.94863, + "75": 9.41807, + "76": 9.08721, + "77": 10.06284, + "78": 9.729, + "79": 9.37087, + "80": 9.40029, + "81": 9.47753, + "82": 9.69123, + "83": 9.30764, + "84": 9.4125, + "85": 9.61132, + "86": 9.07624, + "87": 9.59459, + "88": 9.74769, + "89": 9.60678, + "90": 9.81079, + "91": 9.34443, + "92": 9.36534, + "93": 9.07741, + "94": 8.82974, + "95": 9.51676, + "96": 9.52545, + "97": 9.31031, + "98": 9.67811, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 59.0, + "2": 59.0, + "3": 57.0, + "4": 55.0, + "5": 68.0, + "6": 60.0, + "7": 58.0, + "8": 63.0, + "9": 62.0, + "10": 47.0, + "11": 57.0, + "12": 48.0, + "13": 68.0, + "14": 58.0, + "15": 75.0, + "16": 67.0, + "17": 59.0, + "18": 56.0, + "19": 72.0, + "20": 52.0, + "21": 60.0, + "22": 69.0, + "23": 77.0, + "24": 58.0, + "25": 71.0, + "26": 68.0, + "27": 55.0, + "28": 94.0, + "29": 57.0, + "30": 66.0, + "31": 60.0, + "32": 64.0, + "33": 69.0, + "34": 73.0, + "35": 73.0, + "36": 72.0, + "37": 89.0, + "38": 72.0, + "39": 75.0, + "40": 82.0, + "41": 82.0, + "42": 74.0, + "43": 90.0, + "44": 69.0, + "45": 83.0, + "46": 68.0, + "47": 74.0, + "48": 95.0, + "49": 81.0, + "50": 83.0, + "51": 77.0, + "52": 100.0, + "53": 71.0, + "54": 67.0, + "55": 70.0, + "56": 83.0, + "57": 74.0, + "58": 106.0, + "59": 72.0, + "60": 98.0, + "61": 67.0, + "62": 73.0, + "63": 77.0, + "64": 94.0, + "65": 82.0, + "66": 87.0, + "67": 65.0, + "68": 78.0, + "69": 59.0, + "70": 102.0, + "71": 82.0, + "72": 60.0, + "73": 96.0, + "74": 61.0, + "75": 64.0, + "76": 70.0, + "77": 84.0, + "78": 93.0, + "79": 102.0, + "80": 71.0, + "81": 88.0, + "82": 85.0, + "83": 75.0, + "84": 69.0, + "85": 84.0, + "86": 66.0, + "87": 93.0, + "88": 96.0, + "89": 73.0, + "90": 77.0, + "91": 66.0, + "92": 86.0, + "93": 63.0, + "94": 60.0, + "95": 70.0, + "96": 65.0, + "97": 67.0, + "98": 96.0, + "99": 54.0, + "100": 77.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 516063744.0, + "2": 516063744.0, + "3": 516063744.0, + "4": 516063744.0, + "5": 516063744.0, + "6": 516063744.0, + "7": 516063744.0, + "8": 516063744.0, + "9": 516063744.0, + "10": 516063744.0, + "11": 516063744.0, + "12": 516063744.0, + "13": 516063744.0, + "14": 516063744.0, + "15": 516063744.0, + "16": 516063744.0, + "17": 516063744.0, + "18": 516063744.0, + "19": 516063744.0, + "20": 516063744.0, + "21": 516063744.0, + "22": 516063744.0, + "23": 516063744.0, + "24": 516063744.0, + "25": 516063744.0, + "26": 516063744.0, + "27": 516063744.0, + "28": 516063744.0, + "29": 516063744.0, + "30": 516063744.0, + "31": 516063744.0, + "32": 516063744.0, + "33": 516063744.0, + "34": 516063744.0, + "35": 516063744.0, + "36": 516063744.0, + "37": 516063744.0, + "38": 516063744.0, + "39": 516063744.0, + "40": 516063744.0, + "41": 516063744.0, + "42": 516063744.0, + "43": 516063744.0, + "44": 516063744.0, + "45": 516063744.0, + "46": 516063744.0, + "47": 516063744.0, + "48": 516063744.0, + "49": 516063744.0, + "50": 516063744.0, + "51": 516063744.0, + "52": 516063744.0, + "53": 516063744.0, + "54": 516063744.0, + "55": 516063744.0, + "56": 516063744.0, + "57": 516063744.0, + "58": 516063744.0, + "59": 516063744.0, + "60": 516063744.0, + "61": 516063744.0, + "62": 516063744.0, + "63": 516063744.0, + "64": 516063744.0, + "65": 516063744.0, + "66": 516063744.0, + "67": 516063744.0, + "68": 516063744.0, + "69": 516063744.0, + "70": 516063744.0, + "71": 516063744.0, + "72": 516063744.0, + "73": 516063744.0, + "74": 516063744.0, + "75": 516063744.0, + "76": 516063744.0, + "77": 516063744.0, + "78": 516063744.0, + "79": 516063744.0, + "80": 516063744.0, + "81": 516063744.0, + "82": 516063744.0, + "83": 516063744.0, + "84": 516063744.0, + "85": 516063744.0, + "86": 516063744.0, + "87": 516063744.0, + "88": 516063744.0, + "89": 516063744.0, + "90": 516063744.0, + "91": 516063744.0, + "92": 516063744.0, + "93": 516063744.0, + "94": 516063744.0, + "95": 516063744.0, + "96": 516063744.0, + "97": 516063744.0, + "98": 516063744.0, + "99": 516063744.0, + "100": 516063744.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1697019392.0, + "2": 1877092864.0, + "3": 1877092864.0, + "4": 1877092864.0, + "5": 1877092864.0, + "6": 1877092864.0, + "7": 1877092864.0, + "8": 1877092864.0, + "9": 1877092864.0, + "10": 1877092864.0, + "11": 1877092864.0, + "12": 1877092864.0, + "13": 1877092864.0, + "14": 1877092864.0, + "15": 1877092864.0, + "16": 1877092864.0, + "17": 1877092864.0, + "18": 1877092864.0, + "19": 1877092864.0, + "20": 1877092864.0, + "21": 1877092864.0, + "22": 1877092864.0, + "23": 1877092864.0, + "24": 1877092864.0, + "25": 1877092864.0, + "26": 1877092864.0, + "27": 1877092864.0, + "28": 1877092864.0, + "29": 1877092864.0, + "30": 1877092864.0, + "31": 1877092864.0, + "32": 1877092864.0, + "33": 1877092864.0, + "34": 1877092864.0, + "35": 1877092864.0, + "36": 1877092864.0, + "37": 1877092864.0, + "38": 1877092864.0, + "39": 1877092864.0, + "40": 1877092864.0, + "41": 1877092864.0, + "42": 1877092864.0, + "43": 1877092864.0, + "44": 1877092864.0, + "45": 1877092864.0, + "46": 1877092864.0, + "47": 1877092864.0, + "48": 1877092864.0, + "49": 1877092864.0, + "50": 1877092864.0, + "51": 1877092864.0, + "52": 1877092864.0, + "53": 1877092864.0, + "54": 1877092864.0, + "55": 1877092864.0, + "56": 1877092864.0, + "57": 1877092864.0, + "58": 1877092864.0, + "59": 1877092864.0, + "60": 1877092864.0, + "61": 1877092864.0, + "62": 1877092864.0, + "63": 1877092864.0, + "64": 1877092864.0, + "65": 1877092864.0, + "66": 1877092864.0, + "67": 1877092864.0, + "68": 1877092864.0, + "69": 1877092864.0, + "70": 1877092864.0, + "71": 1877092864.0, + "72": 1877092864.0, + "73": 1877092864.0, + "74": 1877092864.0, + "75": 1877092864.0, + "76": 1877092864.0, + "77": 1877092864.0, + "78": 1877092864.0, + "79": 1877092864.0, + "80": 1877092864.0, + "81": 1877092864.0, + "82": 1877092864.0, + "83": 1877092864.0, + "84": 1877092864.0, + "85": 1877092864.0, + "86": 1877092864.0, + "87": 1877092864.0, + "88": 1877092864.0, + "89": 1877092864.0, + "90": 1877092864.0, + "91": 1877092864.0, + "92": 1877092864.0, + "93": 1877092864.0, + "94": 1877092864.0, + "95": 1877092864.0, + "96": 1877092864.0, + "97": 1877092864.0, + "98": 1877092864.0, + "99": 1877092864.0, + "100": 1877092864.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.10882, + "2": 0.24563, + "3": 0.21507, + "4": 0.21225, + "5": 0.21165, + "6": 0.21127, + "7": 0.21406, + "8": 0.21402, + "9": 0.21175, + "10": 0.19518, + "11": 0.19565, + "12": 0.19461, + "13": 0.19428, + "14": 0.19385, + "15": 0.19329, + "16": 0.19311, + "17": 0.19391, + "18": 0.19383, + "19": 0.19364, + "20": 0.19408, + "21": 0.19327, + "22": 0.19729, + "23": 0.19599, + "24": 0.19601, + "25": 0.1965, + "26": 0.19683, + "27": 0.19626, + "28": 0.19667, + "29": 0.1989, + "30": 0.19644, + "31": 0.19728, + "32": 0.19614, + "33": 0.1973, + "34": 0.1971, + "35": 0.19674, + "36": 0.19628, + "37": 0.19578, + "38": 0.19629, + "39": 0.19673, + "40": 0.19712, + "41": 0.19593, + "42": 0.1969, + "43": 0.19639, + "44": 0.20378, + "45": 0.19737, + "46": 0.19738, + "47": 0.19532, + "48": 0.19579, + "49": 0.19617, + "50": 0.19695, + "51": 0.20318, + "52": 0.19428, + "53": 0.19415, + "54": 0.19663, + "55": 0.19266, + "56": 0.19426, + "57": 0.19455, + "58": 0.19473, + "59": 0.19413, + "60": 0.19467, + "61": 0.19511, + "62": 0.19475, + "63": 0.19464, + "64": 0.19452, + "65": 0.19445, + "66": 0.19395, + "67": 0.19423, + "68": 0.19431, + "69": 0.19512, + "70": 0.1941, + "71": 0.19453, + "72": 0.19467, + "73": 0.19615, + "74": 0.19355, + "75": 0.19419, + "76": 0.19407, + "77": 0.19455, + "78": 0.19511, + "79": 0.19498, + "80": 0.19577, + "81": 0.19399, + "82": 0.19362, + "83": 0.19425, + "84": 0.19418, + "85": 0.19432, + "86": 0.20057, + "87": 0.19522, + "88": 0.19447, + "89": 0.19472, + "90": 0.19377, + "91": 0.19433, + "92": 0.19432, + "93": 0.19456, + "94": 0.19394, + "95": 0.19417, + "96": 0.19476, + "97": 0.19423, + "98": 0.19401, + "99": 0.19403, + "100": 0.19364 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json new file mode 100644 index 00000000000..6a8c9ac9b1d --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 482499072.0, + "2": 482499072.0, + "3": 482499072.0, + "4": 482499072.0, + "5": 482499072.0, + "6": 482499072.0, + "7": 482499072.0, + "8": 482499072.0, + "9": 482499072.0, + "10": 482499072.0, + "11": 482499072.0, + "12": 482499072.0, + "13": 482499072.0, + "14": 482499072.0, + "15": 482499072.0, + "16": 482499072.0, + "17": 482499072.0, + "18": 482499072.0, + "19": 482499072.0, + "20": 482499072.0, + "21": 482499072.0, + "22": 482499072.0, + "23": 482499072.0, + "24": 482499072.0, + "25": 482499072.0, + "26": 482499072.0, + "27": 482499072.0, + "28": 482499072.0, + "29": 482499072.0, + "30": 482499072.0, + "31": 482499072.0, + "32": 482499072.0, + "33": 482499072.0, + "34": 482499072.0, + "35": 482499072.0, + "36": 482499072.0, + "37": 482499072.0, + "38": 482499072.0, + "39": 482499072.0, + "40": 482499072.0, + "41": 482499072.0, + "42": 482499072.0, + "43": 482499072.0, + "44": 482499072.0, + "45": 482499072.0, + "46": 482499072.0, + "47": 482499072.0, + "48": 482499072.0, + "49": 482499072.0, + "50": 482499072.0, + "51": 482499072.0, + "52": 482499072.0, + "53": 482499072.0, + "54": 482499072.0, + "55": 482499072.0, + "56": 482499072.0, + "57": 482499072.0, + "58": 482499072.0, + "59": 482499072.0, + "60": 482499072.0, + "61": 482499072.0, + "62": 482499072.0, + "63": 482499072.0, + "64": 482499072.0, + "65": 482499072.0, + "66": 482499072.0, + "67": 482499072.0, + "68": 482499072.0, + "69": 482499072.0, + "70": 482499072.0, + "71": 482499072.0, + "72": 482499072.0, + "73": 482499072.0, + "74": 482499072.0, + "75": 482499072.0, + "76": 482499072.0, + "77": 482499072.0, + "78": 482499072.0, + "79": 482499072.0, + "80": 482499072.0, + "81": 482499072.0, + "82": 482499072.0, + "83": 482499072.0, + "84": 482499072.0, + "85": 482499072.0, + "86": 482499072.0, + "87": 482499072.0, + "88": 482499072.0, + "89": 482499072.0, + "90": 482499072.0, + "91": 482499072.0, + "92": 482499072.0, + "93": 482499072.0, + "94": 482499072.0, + "95": 482499072.0, + "96": 482499072.0, + "97": 482499072.0, + "98": 482499072.0, + "99": 482499072.0, + "100": 482499072.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1222004224.0, + "2": 1401029120.0, + "3": 1401029120.0, + "4": 1401029120.0, + "5": 1401029120.0, + "6": 1401029120.0, + "7": 1401029120.0, + "8": 1401029120.0, + "9": 1401029120.0, + "10": 1401029120.0, + "11": 1401029120.0, + "12": 1401029120.0, + "13": 1401029120.0, + "14": 1401029120.0, + "15": 1401029120.0, + "16": 1401029120.0, + "17": 1401029120.0, + "18": 1401029120.0, + "19": 1401029120.0, + "20": 1401029120.0, + "21": 1401029120.0, + "22": 1401029120.0, + "23": 1401029120.0, + "24": 1401029120.0, + "25": 1401029120.0, + "26": 1401029120.0, + "27": 1401029120.0, + "28": 1401029120.0, + "29": 1401029120.0, + "30": 1401029120.0, + "31": 1401029120.0, + "32": 1401029120.0, + "33": 1401029120.0, + "34": 1401029120.0, + "35": 1401029120.0, + "36": 1401029120.0, + "37": 1401029120.0, + "38": 1401029120.0, + "39": 1401029120.0, + "40": 1401029120.0, + "41": 1401029120.0, + "42": 1401029120.0, + "43": 1401029120.0, + "44": 1401029120.0, + "45": 1401029120.0, + "46": 1401029120.0, + "47": 1401029120.0, + "48": 1401029120.0, + "49": 1401029120.0, + "50": 1401029120.0, + "51": 1401029120.0, + "52": 1401029120.0, + "53": 1401029120.0, + "54": 1401029120.0, + "55": 1401029120.0, + "56": 1401029120.0, + "57": 1401029120.0, + "58": 1401029120.0, + "59": 1401029120.0, + "60": 1401029120.0, + "61": 1401029120.0, + "62": 1401029120.0, + "63": 1401029120.0, + "64": 1401029120.0, + "65": 1401029120.0, + "66": 1401029120.0, + "67": 1401029120.0, + "68": 1401029120.0, + "69": 1401029120.0, + "70": 1401029120.0, + "71": 1401029120.0, + "72": 1401029120.0, + "73": 1401029120.0, + "74": 1401029120.0, + "75": 1401029120.0, + "76": 1401029120.0, + "77": 1401029120.0, + "78": 1401029120.0, + "79": 1401029120.0, + "80": 1401029120.0, + "81": 1401029120.0, + "82": 1401029120.0, + "83": 1401029120.0, + "84": 1401029120.0, + "85": 1401029120.0, + "86": 1401029120.0, + "87": 1401029120.0, + "88": 1401029120.0, + "89": 1401029120.0, + "90": 1401029120.0, + "91": 1401029120.0, + "92": 1401029120.0, + "93": 1401029120.0, + "94": 1401029120.0, + "95": 1401029120.0, + "96": 1401029120.0, + "97": 1401029120.0, + "98": 1401029120.0, + "99": 1401029120.0, + "100": 1401029120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.51486, + "2": 0.24464, + "3": 0.21825, + "4": 0.21579, + "5": 0.21661, + "6": 0.22001, + "7": 0.22285, + "8": 0.21811, + "9": 0.21904, + "10": 0.22002, + "11": 0.21823, + "12": 0.21823, + "13": 0.21801, + "14": 0.2183, + "15": 0.21803, + "16": 0.21896, + "17": 0.22491, + "18": 0.21905, + "19": 0.21958, + "20": 0.21884, + "21": 0.21909, + "22": 0.21942, + "23": 0.21797, + "24": 0.21855, + "25": 0.21901, + "26": 0.21791, + "27": 0.2177, + "28": 0.2177, + "29": 0.21821, + "30": 0.21868, + "31": 0.21834, + "32": 0.22204, + "33": 0.22091, + "34": 0.21934, + "35": 0.21783, + "36": 0.21796, + "37": 0.21775, + "38": 0.22135, + "39": 0.22172, + "40": 0.21756, + "41": 0.2177, + "42": 0.21859, + "43": 0.21823, + "44": 0.21755, + "45": 0.21785, + "46": 0.21785, + "47": 0.21753, + "48": 0.21811, + "49": 0.21778, + "50": 0.218, + "51": 0.22344, + "52": 0.21422, + "53": 0.21383, + "54": 0.21399, + "55": 0.21397, + "56": 0.21365, + "57": 0.21387, + "58": 0.20817, + "59": 0.20847, + "60": 0.20906, + "61": 0.20895, + "62": 0.20893, + "63": 0.20812, + "64": 0.20751, + "65": 0.20797, + "66": 0.2078, + "67": 0.20815, + "68": 0.20705, + "69": 0.20775, + "70": 0.20774, + "71": 0.2069, + "72": 0.20722, + "73": 0.20718, + "74": 0.2072, + "75": 0.20697, + "76": 0.20716, + "77": 0.20758, + "78": 0.20779, + "79": 0.20721, + "80": 0.20748, + "81": 0.20797, + "82": 0.20844, + "83": 0.20732, + "84": 0.20744, + "85": 0.20706, + "86": 0.20715, + "87": 0.20731, + "88": 0.20708, + "89": 0.20778, + "90": 0.20769, + "91": 0.2074, + "92": 0.20797, + "93": 0.20707, + "94": 0.2072, + "95": 0.20676, + "96": 0.20732, + "97": 0.20728, + "98": 0.20675, + "99": 0.2068, + "100": 0.20724 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json index 339f9b5b061..d6134cdcc5a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92795, "10": 10.90786, "15": 10.88314, "20": 10.77629, "25": 10.59141, "30": 10.39192, "35": 10.29686, "40": 10.0964, "45": 9.84464, "50": 9.90918, "55": 9.87762, "60": 9.49121, "65": 8.94236, "70": 9.72266, "75": 9.41912, "80": 9.40077, "85": 9.61207, "90": 9.81015, "95": 9.51716, "100": 9.4015}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1627.0, "5": 2010.0, "10": 1368.0, "15": 1897.0, "20": 1626.0, "25": 1743.0, "30": 1930.0, "35": 1954.0, "40": 2199.0, "45": 2068.0, "50": 2460.0, "55": 2427.0, "60": 2380.0, "65": 2657.0, "70": 3265.0, "75": 2675.0, "80": 3434.0, "85": 3302.0, "90": 3230.0, "95": 3340.0, "100": 3291.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 486047744.0, "5": 486047744.0, "10": 486047744.0, "15": 486047744.0, "20": 486047744.0, "25": 486047744.0, "30": 486047744.0, "35": 486047744.0, "40": 486047744.0, "45": 486047744.0, "50": 486047744.0, "55": 486047744.0, "60": 486047744.0, "65": 486047744.0, "70": 486047744.0, "75": 486047744.0, "80": 486047744.0, "85": 486047744.0, "90": 486047744.0, "95": 486047744.0, "100": 486047744.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1715890176.0, "5": 1895963648.0, "10": 1895963648.0, "15": 1895963648.0, "20": 1895963648.0, "25": 1895963648.0, "30": 1895963648.0, "35": 1895963648.0, "40": 1895963648.0, "45": 1895963648.0, "50": 1895963648.0, "55": 1895963648.0, "60": 1895963648.0, "65": 1895963648.0, "70": 1895963648.0, "75": 1895963648.0, "80": 1895963648.0, "85": 1895963648.0, "90": 1895963648.0, "95": 1895963648.0, "100": 1895963648.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.92657, "5": 0.21658, "10": 0.20915, "15": 0.2079, "20": 0.21595, "25": 0.21334, "30": 0.21734, "35": 0.21646, "40": 0.21045, "45": 0.20732, "50": 0.2058, "55": 0.20444, "60": 0.20486, "65": 0.20503, "70": 0.20714, "75": 0.20517, "80": 0.2067, "85": 0.20664, "90": 0.20761, "95": 0.20379, "100": 0.20501}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 482499072.0, + "2": 482499072.0, + "3": 482499072.0, + "4": 482499072.0, + "5": 482499072.0, + "6": 482499072.0, + "7": 482499072.0, + "8": 482499072.0, + "9": 482499072.0, + "10": 482499072.0, + "11": 482499072.0, + "12": 482499072.0, + "13": 482499072.0, + "14": 482499072.0, + "15": 482499072.0, + "16": 482499072.0, + "17": 482499072.0, + "18": 482499072.0, + "19": 482499072.0, + "20": 482499072.0, + "21": 482499072.0, + "22": 482499072.0, + "23": 482499072.0, + "24": 482499072.0, + "25": 482499072.0, + "26": 482499072.0, + "27": 482499072.0, + "28": 482499072.0, + "29": 482499072.0, + "30": 482499072.0, + "31": 482499072.0, + "32": 482499072.0, + "33": 482499072.0, + "34": 482499072.0, + "35": 482499072.0, + "36": 482499072.0, + "37": 482499072.0, + "38": 482499072.0, + "39": 482499072.0, + "40": 482499072.0, + "41": 482499072.0, + "42": 482499072.0, + "43": 482499072.0, + "44": 482499072.0, + "45": 482499072.0, + "46": 482499072.0, + "47": 482499072.0, + "48": 482499072.0, + "49": 482499072.0, + "50": 482499072.0, + "51": 482499072.0, + "52": 482499072.0, + "53": 482499072.0, + "54": 482499072.0, + "55": 482499072.0, + "56": 482499072.0, + "57": 482499072.0, + "58": 482499072.0, + "59": 482499072.0, + "60": 482499072.0, + "61": 482499072.0, + "62": 482499072.0, + "63": 482499072.0, + "64": 482499072.0, + "65": 482499072.0, + "66": 482499072.0, + "67": 482499072.0, + "68": 482499072.0, + "69": 482499072.0, + "70": 482499072.0, + "71": 482499072.0, + "72": 482499072.0, + "73": 482499072.0, + "74": 482499072.0, + "75": 482499072.0, + "76": 482499072.0, + "77": 482499072.0, + "78": 482499072.0, + "79": 482499072.0, + "80": 482499072.0, + "81": 482499072.0, + "82": 482499072.0, + "83": 482499072.0, + "84": 482499072.0, + "85": 482499072.0, + "86": 482499072.0, + "87": 482499072.0, + "88": 482499072.0, + "89": 482499072.0, + "90": 482499072.0, + "91": 482499072.0, + "92": 482499072.0, + "93": 482499072.0, + "94": 482499072.0, + "95": 482499072.0, + "96": 482499072.0, + "97": 482499072.0, + "98": 482499072.0, + "99": 482499072.0, + "100": 482499072.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1222004224.0, + "2": 1401029120.0, + "3": 1401029120.0, + "4": 1401029120.0, + "5": 1401029120.0, + "6": 1401029120.0, + "7": 1401029120.0, + "8": 1401029120.0, + "9": 1401029120.0, + "10": 1401029120.0, + "11": 1401029120.0, + "12": 1401029120.0, + "13": 1401029120.0, + "14": 1401029120.0, + "15": 1401029120.0, + "16": 1401029120.0, + "17": 1401029120.0, + "18": 1401029120.0, + "19": 1401029120.0, + "20": 1401029120.0, + "21": 1401029120.0, + "22": 1401029120.0, + "23": 1401029120.0, + "24": 1401029120.0, + "25": 1401029120.0, + "26": 1401029120.0, + "27": 1401029120.0, + "28": 1401029120.0, + "29": 1401029120.0, + "30": 1401029120.0, + "31": 1401029120.0, + "32": 1401029120.0, + "33": 1401029120.0, + "34": 1401029120.0, + "35": 1401029120.0, + "36": 1401029120.0, + "37": 1401029120.0, + "38": 1401029120.0, + "39": 1401029120.0, + "40": 1401029120.0, + "41": 1401029120.0, + "42": 1401029120.0, + "43": 1401029120.0, + "44": 1401029120.0, + "45": 1401029120.0, + "46": 1401029120.0, + "47": 1401029120.0, + "48": 1401029120.0, + "49": 1401029120.0, + "50": 1401029120.0, + "51": 1401029120.0, + "52": 1401029120.0, + "53": 1401029120.0, + "54": 1401029120.0, + "55": 1401029120.0, + "56": 1401029120.0, + "57": 1401029120.0, + "58": 1401029120.0, + "59": 1401029120.0, + "60": 1401029120.0, + "61": 1401029120.0, + "62": 1401029120.0, + "63": 1401029120.0, + "64": 1401029120.0, + "65": 1401029120.0, + "66": 1401029120.0, + "67": 1401029120.0, + "68": 1401029120.0, + "69": 1401029120.0, + "70": 1401029120.0, + "71": 1401029120.0, + "72": 1401029120.0, + "73": 1401029120.0, + "74": 1401029120.0, + "75": 1401029120.0, + "76": 1401029120.0, + "77": 1401029120.0, + "78": 1401029120.0, + "79": 1401029120.0, + "80": 1401029120.0, + "81": 1401029120.0, + "82": 1401029120.0, + "83": 1401029120.0, + "84": 1401029120.0, + "85": 1401029120.0, + "86": 1401029120.0, + "87": 1401029120.0, + "88": 1401029120.0, + "89": 1401029120.0, + "90": 1401029120.0, + "91": 1401029120.0, + "92": 1401029120.0, + "93": 1401029120.0, + "94": 1401029120.0, + "95": 1401029120.0, + "96": 1401029120.0, + "97": 1401029120.0, + "98": 1401029120.0, + "99": 1401029120.0, + "100": 1401029120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.20377, + "2": 0.2288, + "3": 0.19616, + "4": 0.19587, + "5": 0.19737, + "6": 0.19775, + "7": 0.19658, + "8": 0.19621, + "9": 0.19557, + "10": 0.19534, + "11": 0.19453, + "12": 0.1949, + "13": 0.19522, + "14": 0.19865, + "15": 0.20415, + "16": 0.19686, + "17": 0.1985, + "18": 0.19858, + "19": 0.19709, + "20": 0.19609, + "21": 0.19758, + "22": 0.19837, + "23": 0.19786, + "24": 0.19688, + "25": 0.1972, + "26": 0.19859, + "27": 0.19814, + "28": 0.1989, + "29": 0.1984, + "30": 0.19783, + "31": 0.19727, + "32": 0.19754, + "33": 0.19648, + "34": 0.19977, + "35": 0.19847, + "36": 0.19696, + "37": 0.20498, + "38": 0.20415, + "39": 0.20225, + "40": 0.19712, + "41": 0.19751, + "42": 0.19764, + "43": 0.19738, + "44": 0.19703, + "45": 0.19703, + "46": 0.19814, + "47": 0.19757, + "48": 0.19759, + "49": 0.19688, + "50": 0.20181, + "51": 0.22215, + "52": 0.2134, + "53": 0.2129, + "54": 0.2133, + "55": 0.21255, + "56": 0.21221, + "57": 0.21233, + "58": 0.2124, + "59": 0.21242, + "60": 0.21258, + "61": 0.21219, + "62": 0.21255, + "63": 0.21385, + "64": 0.2127, + "65": 0.21252, + "66": 0.21191, + "67": 0.21327, + "68": 0.21176, + "69": 0.2127, + "70": 0.21284, + "71": 0.21291, + "72": 0.21265, + "73": 0.21221, + "74": 0.21387, + "75": 0.21247, + "76": 0.21204, + "77": 0.21169, + "78": 0.21259, + "79": 0.21196, + "80": 0.21204, + "81": 0.21211, + "82": 0.21314, + "83": 0.21268, + "84": 0.21291, + "85": 0.21328, + "86": 0.2128, + "87": 0.21213, + "88": 0.21192, + "89": 0.21242, + "90": 0.21253, + "91": 0.21252, + "92": 0.21236, + "93": 0.21254, + "94": 0.21255, + "95": 0.21209, + "96": 0.21345, + "97": 0.21202, + "98": 0.21234, + "99": 0.21237, + "100": 0.21317 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..b276f849ced --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.85961, + "2": 10.85774, + "3": 10.85857, + "4": 10.84879, + "5": 10.88247, + "6": 10.88701, + "7": 10.86152, + "8": 10.86792, + "9": 10.86512, + "10": 10.83742, + "11": 10.87758, + "12": 10.87232, + "13": 10.87849, + "14": 10.88848, + "15": 10.82866, + "16": 10.8307, + "17": 10.79817, + "18": 10.82417, + "19": 10.81181, + "20": 10.72691, + "21": 10.7183, + "22": 10.57059, + "23": 10.72817, + "24": 10.61403, + "25": 10.55842, + "26": 10.60676, + "27": 10.61661, + "28": 10.57482, + "29": 10.59587, + "30": 10.37815, + "31": 10.13027, + "32": 10.47778, + "33": 10.46854, + "34": 10.22658, + "35": 10.27984, + "36": 10.22842, + "37": 10.35309, + "38": 10.20319, + "39": 10.4096, + "40": 10.0888, + "41": 10.15947, + "42": 10.2147, + "43": 9.84971, + "44": 9.96501, + "45": 9.82545, + "46": 9.83484, + "47": 10.15515, + "48": 9.84462, + "49": 9.52863, + "50": 9.91253 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1816.0, + "2": 1884.0, + "3": 1906.0, + "4": 1901.0, + "5": 2061.0, + "6": 2186.0, + "7": 2145.0, + "8": 1775.0, + "9": 2106.0, + "10": 1630.0, + "11": 2236.0, + "12": 2015.0, + "13": 2168.0, + "14": 2021.0, + "15": 2088.0, + "16": 2152.0, + "17": 1976.0, + "18": 1952.0, + "19": 2046.0, + "20": 1740.0, + "21": 2073.0, + "22": 1810.0, + "23": 2186.0, + "24": 1839.0, + "25": 1681.0, + "26": 1962.0, + "27": 2009.0, + "28": 2204.0, + "29": 2264.0, + "30": 2117.0, + "31": 1873.0, + "32": 2178.0, + "33": 2372.0, + "34": 2127.0, + "35": 2225.0, + "36": 2140.0, + "37": 2649.0, + "38": 2602.0, + "39": 2607.0, + "40": 2574.0, + "41": 2503.0, + "42": 2532.0, + "43": 2285.0, + "44": 2313.0, + "45": 2423.0, + "46": 2563.0, + "47": 2856.0, + "48": 2629.0, + "49": 2566.0, + "50": 2651.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1449634304.0, + "2": 1516021248.0, + "3": 1516021248.0, + "4": 1516021248.0, + "5": 1516021248.0, + "6": 1516021248.0, + "7": 1516021248.0, + "8": 1516021248.0, + "9": 1516021248.0, + "10": 1516021248.0, + "11": 1516021248.0, + "12": 1516021248.0, + "13": 1516021248.0, + "14": 1516021248.0, + "15": 1516021248.0, + "16": 1516021248.0, + "17": 1516021248.0, + "18": 1516021248.0, + "19": 1516021248.0, + "20": 1516021248.0, + "21": 1516021248.0, + "22": 1516021248.0, + "23": 1516021248.0, + "24": 1516021248.0, + "25": 1516021248.0, + "26": 1516021248.0, + "27": 1516021248.0, + "28": 1516021248.0, + "29": 1516021248.0, + "30": 1516021248.0, + "31": 1516021248.0, + "32": 1516021248.0, + "33": 1516021248.0, + "34": 1516021248.0, + "35": 1516021248.0, + "36": 1516021248.0, + "37": 1516021248.0, + "38": 1516021248.0, + "39": 1516021248.0, + "40": 1516021248.0, + "41": 1516021248.0, + "42": 1516021248.0, + "43": 1516021248.0, + "44": 1516021248.0, + "45": 1516021248.0, + "46": 1516021248.0, + "47": 1516021248.0, + "48": 1516021248.0, + "49": 1516021248.0, + "50": 1516021248.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.56269, + "2": 0.39732, + "3": 0.33618, + "4": 0.33333, + "5": 0.33363, + "6": 0.33477, + "7": 0.33191, + "8": 0.33146, + "9": 0.33302, + "10": 0.33134, + "11": 0.33151, + "12": 0.33246, + "13": 0.33088, + "14": 0.3303, + "15": 0.33382, + "16": 0.33069, + "17": 0.33094, + "18": 0.33118, + "19": 0.33644, + "20": 0.3305, + "21": 0.33088, + "22": 0.33128, + "23": 0.33088, + "24": 0.33099, + "25": 0.33033, + "26": 0.33052, + "27": 0.33152, + "28": 0.33036, + "29": 0.33036, + "30": 0.33116, + "31": 0.33249, + "32": 0.33284, + "33": 0.33031, + "34": 0.33138, + "35": 0.33068, + "36": 0.33706, + "37": 0.33218, + "38": 0.33033, + "39": 0.33382, + "40": 0.34395, + "41": 0.33213, + "42": 0.3316, + "43": 0.33261, + "44": 0.33133, + "45": 0.33097, + "46": 0.33137, + "47": 0.33245, + "48": 0.33171, + "49": 0.33138, + "50": 0.33185 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index 28f4a9f3d58..5dd18b2b701 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.86122, "5": 10.88242, "10": 10.83506, "15": 10.82738, "20": 10.72743, "25": 10.55753, "30": 10.37895, "35": 10.28321, "40": 10.08785, "45": 9.82625, "50": 9.91327}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1778.0, "5": 2219.0, "10": 1530.0, "15": 2125.0, "20": 1867.0, "25": 1773.0, "30": 2129.0, "35": 2169.0, "40": 2486.0, "45": 2335.0, "50": 2687.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 269891584.0, "5": 269891584.0, "10": 269891584.0, "15": 269891584.0, "20": 269891584.0, "25": 269891584.0, "30": 269891584.0, "35": 269891584.0, "40": 269891584.0, "45": 269891584.0, "50": 269891584.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1450730496.0, "5": 1515676160.0, "10": 1515676672.0, "15": 1515676672.0, "20": 1515676672.0, "25": 1515676672.0, "30": 1515676672.0, "35": 1515676672.0, "40": 1515676672.0, "45": 1515676672.0, "50": 1515676672.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 11.10997, "5": 0.31331, "10": 0.32223, "15": 0.31308, "20": 0.3166, "25": 0.31574, "30": 0.31074, "35": 0.31512, "40": 0.31604, "45": 0.32221, "50": 0.31241}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.85961, + "2": 10.85774, + "3": 10.85857, + "4": 10.84879, + "5": 10.88247, + "6": 10.88701, + "7": 10.86152, + "8": 10.86792, + "9": 10.86512, + "10": 10.83742, + "11": 10.87758, + "12": 10.87232, + "13": 10.87849, + "14": 10.88848, + "15": 10.82866, + "16": 10.8307, + "17": 10.79817, + "18": 10.82417, + "19": 10.81181, + "20": 10.72691, + "21": 10.7183, + "22": 10.57059, + "23": 10.72817, + "24": 10.61403, + "25": 10.55842, + "26": 10.60676, + "27": 10.61661, + "28": 10.57482, + "29": 10.59587, + "30": 10.37815, + "31": 10.13027, + "32": 10.47778, + "33": 10.46854, + "34": 10.22658, + "35": 10.27984, + "36": 10.22842, + "37": 10.35309, + "38": 10.20319, + "39": 10.4096, + "40": 10.0888, + "41": 10.15947, + "42": 10.2147, + "43": 9.84971, + "44": 9.96501, + "45": 9.82545, + "46": 9.83484, + "47": 10.15515, + "48": 9.84462, + "49": 9.52863, + "50": 9.91253 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1816.0, + "2": 1884.0, + "3": 1906.0, + "4": 1901.0, + "5": 2061.0, + "6": 2186.0, + "7": 2145.0, + "8": 1775.0, + "9": 2106.0, + "10": 1630.0, + "11": 2236.0, + "12": 2015.0, + "13": 2168.0, + "14": 2021.0, + "15": 2088.0, + "16": 2152.0, + "17": 1976.0, + "18": 1952.0, + "19": 2046.0, + "20": 1740.0, + "21": 2073.0, + "22": 1810.0, + "23": 2186.0, + "24": 1839.0, + "25": 1681.0, + "26": 1962.0, + "27": 2009.0, + "28": 2204.0, + "29": 2264.0, + "30": 2117.0, + "31": 1873.0, + "32": 2178.0, + "33": 2372.0, + "34": 2127.0, + "35": 2225.0, + "36": 2140.0, + "37": 2649.0, + "38": 2602.0, + "39": 2607.0, + "40": 2574.0, + "41": 2503.0, + "42": 2532.0, + "43": 2285.0, + "44": 2313.0, + "45": 2423.0, + "46": 2563.0, + "47": 2856.0, + "48": 2629.0, + "49": 2566.0, + "50": 2651.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1449634304.0, + "2": 1516021248.0, + "3": 1516021248.0, + "4": 1516021248.0, + "5": 1516021248.0, + "6": 1516021248.0, + "7": 1516021248.0, + "8": 1516021248.0, + "9": 1516021248.0, + "10": 1516021248.0, + "11": 1516021248.0, + "12": 1516021248.0, + "13": 1516021248.0, + "14": 1516021248.0, + "15": 1516021248.0, + "16": 1516021248.0, + "17": 1516021248.0, + "18": 1516021248.0, + "19": 1516021248.0, + "20": 1516021248.0, + "21": 1516021248.0, + "22": 1516021248.0, + "23": 1516021248.0, + "24": 1516021248.0, + "25": 1516021248.0, + "26": 1516021248.0, + "27": 1516021248.0, + "28": 1516021248.0, + "29": 1516021248.0, + "30": 1516021248.0, + "31": 1516021248.0, + "32": 1516021248.0, + "33": 1516021248.0, + "34": 1516021248.0, + "35": 1516021248.0, + "36": 1516021248.0, + "37": 1516021248.0, + "38": 1516021248.0, + "39": 1516021248.0, + "40": 1516021248.0, + "41": 1516021248.0, + "42": 1516021248.0, + "43": 1516021248.0, + "44": 1516021248.0, + "45": 1516021248.0, + "46": 1516021248.0, + "47": 1516021248.0, + "48": 1516021248.0, + "49": 1516021248.0, + "50": 1516021248.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.8193, + "2": 0.36983, + "3": 0.32405, + "4": 0.32179, + "5": 0.32037, + "6": 0.32162, + "7": 0.32479, + "8": 0.32031, + "9": 0.32398, + "10": 0.32296, + "11": 0.32125, + "12": 0.32185, + "13": 0.323, + "14": 0.32307, + "15": 0.32035, + "16": 0.31953, + "17": 0.32119, + "18": 0.32536, + "19": 0.32368, + "20": 0.32071, + "21": 0.32043, + "22": 0.32093, + "23": 0.32096, + "24": 0.31999, + "25": 0.32046, + "26": 0.31988, + "27": 0.32184, + "28": 0.32107, + "29": 0.32078, + "30": 0.32174, + "31": 0.32345, + "32": 0.32975, + "33": 0.32181, + "34": 0.32294, + "35": 0.32426, + "36": 0.32184, + "37": 0.32175, + "38": 0.32222, + "39": 0.32058, + "40": 0.32111, + "41": 0.33546, + "42": 0.32505, + "43": 0.32502, + "44": 0.32486, + "45": 0.32683, + "46": 0.32331, + "47": 0.322, + "48": 0.32205, + "49": 0.32128, + "50": 0.32053 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..e2187c31d21 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971620864.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.72292, + "2": 0.40079, + "3": 0.30233, + "4": 0.29437, + "5": 0.29362, + "6": 0.29166, + "7": 0.29282, + "8": 0.29142, + "9": 0.29651, + "10": 0.29169, + "11": 0.29163, + "12": 0.29014, + "13": 0.29007, + "14": 0.28821, + "15": 0.28623, + "16": 0.28688, + "17": 0.28583, + "18": 0.2859, + "19": 0.28593, + "20": 0.28532, + "21": 0.28573, + "22": 0.28605, + "23": 0.2847, + "24": 0.28621, + "25": 0.28664, + "26": 0.28556, + "27": 0.2872, + "28": 0.28573, + "29": 0.28695, + "30": 0.28636, + "31": 0.28686, + "32": 0.28621, + "33": 0.28593, + "34": 0.28506, + "35": 0.28606, + "36": 0.28626, + "37": 0.28499, + "38": 0.28674, + "39": 0.28971, + "40": 0.28955, + "41": 0.28907, + "42": 0.29041, + "43": 0.28912, + "44": 0.28905, + "45": 0.28882, + "46": 0.28956, + "47": 0.28907, + "48": 0.28954, + "49": 0.28925, + "50": 0.28916, + "51": 0.2978, + "52": 0.29215, + "53": 0.30386, + "54": 0.28798, + "55": 0.30697, + "56": 0.32648, + "57": 0.29076, + "58": 0.29115, + "59": 0.29247, + "60": 0.29092, + "61": 0.29317, + "62": 0.28506, + "63": 0.28662, + "64": 0.2855, + "65": 0.28592, + "66": 0.28679, + "67": 0.28596, + "68": 0.28605, + "69": 0.28703, + "70": 0.28617, + "71": 0.28619, + "72": 0.28686, + "73": 0.28763, + "74": 0.28583, + "75": 0.28621, + "76": 0.28619, + "77": 0.28487, + "78": 0.28523, + "79": 0.28634, + "80": 0.28584, + "81": 0.28564, + "82": 0.28528, + "83": 0.28536, + "84": 0.28516, + "85": 0.28552, + "86": 0.28537, + "87": 0.2846, + "88": 0.28517, + "89": 0.28625, + "90": 0.28499, + "91": 0.28525, + "92": 0.28549, + "93": 0.28511, + "94": 0.28501, + "95": 0.28642, + "96": 0.28514, + "97": 0.28481, + "98": 0.2852, + "99": 0.28627, + "100": 0.2861 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json index 82a2437ad9d..0cc3719ac53 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86122, "5": 10.88242, "10": 10.83506, "15": 10.82738, "20": 10.72743, "25": 10.55753, "30": 10.37895, "35": 10.28321, "40": 10.08785, "45": 9.82625, "50": 9.91327, "55": 9.87788, "60": 9.50874, "65": 8.95103, "70": 9.73167, "75": 9.43681, "80": 9.41156, "85": 9.61613, "90": 9.8217, "95": 9.5191, "100": 9.40588}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1778.0, "5": 2219.0, "10": 1530.0, "15": 2125.0, "20": 1867.0, "25": 1773.0, "30": 2129.0, "35": 2169.0, "40": 2486.0, "45": 2335.0, "50": 2687.0, "55": 2652.0, "60": 2765.0, "65": 2946.0, "70": 3629.0, "75": 2702.0, "80": 3866.0, "85": 3517.0, "90": 3349.0, "95": 3530.0, "100": 3530.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 269891584.0, "5": 269891584.0, "10": 269891584.0, "15": 269891584.0, "20": 269891584.0, "25": 269891584.0, "30": 269891584.0, "35": 269891584.0, "40": 269891584.0, "45": 269891584.0, "50": 269891584.0, "55": 269891584.0, "60": 269891584.0, "65": 269891584.0, "70": 269891584.0, "75": 269891584.0, "80": 269891584.0, "85": 269891584.0, "90": 269891584.0, "95": 269891584.0, "100": 269891584.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1448634368.0, "5": 1515676672.0, "10": 1515676672.0, "15": 1515676672.0, "20": 1515676672.0, "25": 1515676672.0, "30": 1515676672.0, "35": 1515676672.0, "40": 1515676672.0, "45": 1515676672.0, "50": 1515676672.0, "55": 1515676672.0, "60": 1515676672.0, "65": 1515676672.0, "70": 1515676672.0, "75": 1515676672.0, "80": 1515676672.0, "85": 1515676672.0, "90": 1515676672.0, "95": 1515676672.0, "100": 1515676672.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.54144, "5": 0.31998, "10": 0.30758, "15": 0.30768, "20": 0.30227, "25": 0.30186, "30": 0.3093, "35": 0.3126, "40": 0.31097, "45": 0.30984, "50": 0.3102, "55": 0.30366, "60": 0.30466, "65": 0.3097, "70": 0.30131, "75": 0.30287, "80": 0.30479, "85": 0.30276, "90": 0.3042, "95": 0.30274, "100": 0.30335}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971619840.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.22987, + "2": 0.54363, + "3": 0.2879, + "4": 0.28745, + "5": 0.28509, + "6": 0.28364, + "7": 0.28401, + "8": 0.28235, + "9": 0.28321, + "10": 0.32258, + "11": 0.28697, + "12": 0.27808, + "13": 0.27857, + "14": 0.27833, + "15": 0.28035, + "16": 0.27859, + "17": 0.27841, + "18": 0.27879, + "19": 0.27874, + "20": 0.27806, + "21": 0.27812, + "22": 0.2783, + "23": 0.27919, + "24": 0.27841, + "25": 0.27852, + "26": 0.27871, + "27": 0.27891, + "28": 0.28056, + "29": 0.27909, + "30": 0.2797, + "31": 0.27903, + "32": 0.27895, + "33": 0.27929, + "34": 0.27838, + "35": 0.27904, + "36": 0.2787, + "37": 0.28662, + "38": 0.27812, + "39": 0.27805, + "40": 0.27846, + "41": 0.27884, + "42": 0.27807, + "43": 0.27794, + "44": 0.27825, + "45": 0.28052, + "46": 0.27856, + "47": 0.27832, + "48": 0.27799, + "49": 0.2783, + "50": 0.27861, + "51": 0.2915, + "52": 0.28668, + "53": 0.28545, + "54": 0.28632, + "55": 0.28616, + "56": 0.28735, + "57": 0.28738, + "58": 0.28556, + "59": 0.28453, + "60": 0.28543, + "61": 0.28452, + "62": 0.28404, + "63": 0.28542, + "64": 0.28492, + "65": 0.28488, + "66": 0.2861, + "67": 0.286, + "68": 0.28505, + "69": 0.28531, + "70": 0.28377, + "71": 0.28517, + "72": 0.28454, + "73": 0.2853, + "74": 0.28678, + "75": 0.28484, + "76": 0.28523, + "77": 0.28548, + "78": 0.28488, + "79": 0.28559, + "80": 0.28528, + "81": 0.28479, + "82": 0.28465, + "83": 0.28506, + "84": 0.28493, + "85": 0.28486, + "86": 0.28572, + "87": 0.28404, + "88": 0.28473, + "89": 0.28431, + "90": 0.28945, + "91": 0.28446, + "92": 0.28489, + "93": 0.28474, + "94": 0.28484, + "95": 0.28526, + "96": 0.28573, + "97": 0.28411, + "98": 0.28402, + "99": 0.28413, + "100": 0.28454 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..a39e093badf --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971619840.0, + "2": 1036172288.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.83564, + "2": 0.46115, + "3": 0.29577, + "4": 0.29295, + "5": 0.2937, + "6": 0.29159, + "7": 0.29645, + "8": 0.3016, + "9": 0.29961, + "10": 0.29819, + "11": 0.29865, + "12": 0.2945, + "13": 0.29585, + "14": 0.29535, + "15": 0.29488, + "16": 0.29419, + "17": 0.30001, + "18": 0.29304, + "19": 0.29268, + "20": 0.29243, + "21": 0.29254, + "22": 0.29287, + "23": 0.2907, + "24": 0.29139, + "25": 0.29157, + "26": 0.29169, + "27": 0.29335, + "28": 0.29054, + "29": 0.29193, + "30": 0.29255, + "31": 0.29123, + "32": 0.29014, + "33": 0.29168, + "34": 0.29286, + "35": 0.29078, + "36": 0.29177, + "37": 0.29116, + "38": 0.29148, + "39": 0.29215, + "40": 0.2925, + "41": 0.29112, + "42": 0.29245, + "43": 0.28949, + "44": 0.28901, + "45": 0.28833, + "46": 0.28915, + "47": 0.28932, + "48": 0.28813, + "49": 0.28806, + "50": 0.28818, + "51": 0.30469, + "52": 0.29982, + "53": 0.29764, + "54": 0.29724, + "55": 0.29321, + "56": 0.29333, + "57": 0.29533, + "58": 0.29391, + "59": 0.29338, + "60": 0.29347, + "61": 0.29437, + "62": 0.29327, + "63": 0.29354, + "64": 0.31415, + "65": 0.2977, + "66": 0.2961, + "67": 0.29576, + "68": 0.2944, + "69": 0.29403, + "70": 0.29434, + "71": 0.29505, + "72": 0.2967, + "73": 0.30974, + "74": 0.2985, + "75": 0.29801, + "76": 0.29505, + "77": 0.29554, + "78": 0.29652, + "79": 0.29594, + "80": 0.2958, + "81": 0.29561, + "82": 0.29502, + "83": 0.29476, + "84": 0.29486, + "85": 0.29516, + "86": 0.29553, + "87": 0.29666, + "88": 0.29626, + "89": 0.29583, + "90": 0.29691, + "91": 0.29506, + "92": 0.29865, + "93": 0.29628, + "94": 0.30563, + "95": 0.29582, + "96": 0.29572, + "97": 0.29519, + "98": 0.29708, + "99": 0.29725, + "100": 0.29684 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index 54bb3cbea8d..1641ae309dc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.86122, - "2": 10.85774, - "3": 10.86039, - "4": 10.84813, - "5": 10.88242, - "6": 10.88645, - "7": 10.86227, - "8": 10.86932, - "9": 10.86444, - "10": 10.83506, - "11": 10.87765, - "12": 10.87384, - "13": 10.87945, - "14": 10.88919, - "15": 10.82738, - "16": 10.83105, - "17": 10.79888, - "18": 10.82441, - "19": 10.81363, - "20": 10.72743, - "21": 10.71638, - "22": 10.57153, - "23": 10.7269, - "24": 10.61223, - "25": 10.55753, - "26": 10.60603, - "27": 10.61792, - "28": 10.57695, - "29": 10.59633, - "30": 10.37895, - "31": 10.13125, - "32": 10.47822, - "33": 10.46894, - "34": 10.22715, - "35": 10.28321, - "36": 10.22751, - "37": 10.35397, - "38": 10.20483, - "39": 10.40755, - "40": 10.08785, - "41": 10.1591, - "42": 10.21601, - "43": 9.84821, - "44": 9.9651, - "45": 9.82625, - "46": 9.83468, - "47": 10.15337, - "48": 9.84529, - "49": 9.52926, - "50": 9.91327, - "51": 9.8517, - "52": 9.74686, - "53": 10.07204, - "54": 9.95738, - "55": 9.87788, - "56": 9.62943, - "57": 9.48988, - "58": 9.83265, - "59": 9.58831, - "60": 9.50874, - "61": 9.69495, - "62": 9.99373, - "63": 9.377, - "64": 9.78004, - "65": 8.95103, - "66": 9.71392, + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, "67": 9.37884, - "68": 9.78831, - "69": 9.79096, - "70": 9.73167, - "71": 9.61776, - "72": 9.59099, - "73": 9.49436, - "74": 8.95001, - "75": 9.43681, - "76": 9.09852, - "77": 10.06447, - "78": 9.72944, - "79": 9.37805, - "80": 9.41156, - "81": 9.48537, - "82": 9.69592, - "83": 9.31981, - "84": 9.42306, - "85": 9.61613, - "86": 9.07185, - "87": 9.59282, - "88": 9.75055, - "89": 9.61194, - "90": 9.8217, - "91": 9.35308, - "92": 9.36305, - "93": 9.08788, - "94": 8.83439, - "95": 9.5191, - "96": 9.52647, - "97": 9.31412, - "98": 9.67541, - "99": 8.88941, - "100": 9.40588 + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1778.0, - "2": 1875.0, - "3": 1879.0, - "4": 1912.0, - "5": 2219.0, - "6": 2163.0, - "7": 2113.0, - "8": 1747.0, - "9": 2049.0, - "10": 1530.0, - "11": 2113.0, - "12": 1959.0, - "13": 2134.0, - "14": 2055.0, - "15": 2125.0, - "16": 2139.0, - "17": 1988.0, - "18": 1892.0, - "19": 1991.0, - "20": 1867.0, - "21": 2023.0, - "22": 1865.0, - "23": 2185.0, - "24": 1774.0, - "25": 1773.0, - "26": 1990.0, - "27": 2061.0, - "28": 2215.0, - "29": 2186.0, - "30": 2129.0, - "31": 1794.0, - "32": 2109.0, - "33": 2422.0, - "34": 2135.0, - "35": 2169.0, - "36": 2127.0, - "37": 2432.0, - "38": 2490.0, - "39": 2495.0, - "40": 2486.0, - "41": 2465.0, - "42": 2535.0, - "43": 2216.0, - "44": 2407.0, - "45": 2335.0, - "46": 2617.0, - "47": 2830.0, - "48": 2480.0, - "49": 2492.0, - "50": 2687.0, - "51": 2863.0, - "52": 2881.0, - "53": 3220.0, - "54": 2894.0, - "55": 2652.0, - "56": 3006.0, - "57": 2561.0, - "58": 3273.0, - "59": 3039.0, - "60": 2765.0, - "61": 3310.0, - "62": 2936.0, - "63": 2630.0, - "64": 3230.0, - "65": 2946.0, - "66": 3500.0, - "67": 2976.0, - "68": 2944.0, - "69": 3117.0, - "70": 3629.0, - "71": 3255.0, - "72": 2633.0, - "73": 3338.0, - "74": 2172.0, - "75": 2702.0, - "76": 3162.0, - "77": 3850.0, - "78": 3590.0, - "79": 3658.0, - "80": 3866.0, - "81": 3976.0, - "82": 3680.0, - "83": 3153.0, - "84": 3586.0, - "85": 3517.0, - "86": 3137.0, - "87": 4177.0, - "88": 3589.0, - "89": 3849.0, - "90": 3349.0, - "91": 2936.0, - "92": 3526.0, - "93": 2965.0, - "94": 3772.0, - "95": 3530.0, - "96": 3774.0, - "97": 3636.0, - "98": 4064.0, - "99": 3394.0, - "100": 3530.0 + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 269891584.0, - "2": 269891584.0, - "3": 269891584.0, - "4": 269891584.0, - "5": 269891584.0, - "6": 269891584.0, - "7": 269891584.0, - "8": 269891584.0, - "9": 269891584.0, - "10": 269891584.0, - "11": 269891584.0, - "12": 269891584.0, - "13": 269891584.0, - "14": 269891584.0, - "15": 269891584.0, - "16": 269891584.0, - "17": 269891584.0, - "18": 269891584.0, - "19": 269891584.0, - "20": 269891584.0, - "21": 269891584.0, - "22": 269891584.0, - "23": 269891584.0, - "24": 269891584.0, - "25": 269891584.0, - "26": 269891584.0, - "27": 269891584.0, - "28": 269891584.0, - "29": 269891584.0, - "30": 269891584.0, - "31": 269891584.0, - "32": 269891584.0, - "33": 269891584.0, - "34": 269891584.0, - "35": 269891584.0, - "36": 269891584.0, - "37": 269891584.0, - "38": 269891584.0, - "39": 269891584.0, - "40": 269891584.0, - "41": 269891584.0, - "42": 269891584.0, - "43": 269891584.0, - "44": 269891584.0, - "45": 269891584.0, - "46": 269891584.0, - "47": 269891584.0, - "48": 269891584.0, - "49": 269891584.0, - "50": 269891584.0, - "51": 269891584.0, - "52": 269891584.0, - "53": 269891584.0, - "54": 269891584.0, - "55": 269891584.0, - "56": 269891584.0, - "57": 269891584.0, - "58": 269891584.0, - "59": 269891584.0, - "60": 269891584.0, - "61": 269891584.0, - "62": 269891584.0, - "63": 269891584.0, - "64": 269891584.0, - "65": 269891584.0, - "66": 269891584.0, - "67": 269891584.0, - "68": 269891584.0, - "69": 269891584.0, - "70": 269891584.0, - "71": 269891584.0, - "72": 269891584.0, - "73": 269891584.0, - "74": 269891584.0, - "75": 269891584.0, - "76": 269891584.0, - "77": 269891584.0, - "78": 269891584.0, - "79": 269891584.0, - "80": 269891584.0, - "81": 269891584.0, - "82": 269891584.0, - "83": 269891584.0, - "84": 269891584.0, - "85": 269891584.0, - "86": 269891584.0, - "87": 269891584.0, - "88": 269891584.0, - "89": 269891584.0, - "90": 269891584.0, - "91": 269891584.0, - "92": 269891584.0, - "93": 269891584.0, - "94": 269891584.0, - "95": 269891584.0, - "96": 269891584.0, - "97": 269891584.0, - "98": 269891584.0, - "99": 269891584.0, - "100": 269891584.0 + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1448634368.0, - "2": 1515676672.0, - "3": 1515676672.0, - "4": 1515676672.0, - "5": 1515676672.0, - "6": 1515676672.0, - "7": 1515676672.0, - "8": 1515676672.0, - "9": 1515676672.0, - "10": 1515676672.0, - "11": 1515676672.0, - "12": 1515676672.0, - "13": 1515676672.0, - "14": 1515676672.0, - "15": 1515676672.0, - "16": 1515676672.0, - "17": 1515676672.0, - "18": 1515676672.0, - "19": 1515676672.0, - "20": 1515676672.0, - "21": 1515676672.0, - "22": 1515676672.0, - "23": 1515676672.0, - "24": 1515676672.0, - "25": 1515676672.0, - "26": 1515676672.0, - "27": 1515676672.0, - "28": 1515676672.0, - "29": 1515676672.0, - "30": 1515676672.0, - "31": 1515676672.0, - "32": 1515676672.0, - "33": 1515676672.0, - "34": 1515676672.0, - "35": 1515676672.0, - "36": 1515676672.0, - "37": 1515676672.0, - "38": 1515676672.0, - "39": 1515676672.0, - "40": 1515676672.0, - "41": 1515676672.0, - "42": 1515676672.0, - "43": 1515676672.0, - "44": 1515676672.0, - "45": 1515676672.0, - "46": 1515676672.0, - "47": 1515676672.0, - "48": 1515676672.0, - "49": 1515676672.0, - "50": 1515676672.0, - "51": 1515676672.0, - "52": 1515676672.0, - "53": 1515676672.0, - "54": 1515676672.0, - "55": 1515676672.0, - "56": 1515676672.0, - "57": 1515676672.0, - "58": 1515676672.0, - "59": 1515676672.0, - "60": 1515676672.0, - "61": 1515676672.0, - "62": 1515676672.0, - "63": 1515676672.0, - "64": 1515676672.0, - "65": 1515676672.0, - "66": 1515676672.0, - "67": 1515676672.0, - "68": 1515676672.0, - "69": 1515676672.0, - "70": 1515676672.0, - "71": 1515676672.0, - "72": 1515676672.0, - "73": 1515676672.0, - "74": 1515676672.0, - "75": 1515676672.0, - "76": 1515676672.0, - "77": 1515676672.0, - "78": 1515676672.0, - "79": 1515676672.0, - "80": 1515676672.0, - "81": 1515676672.0, - "82": 1515676672.0, - "83": 1515676672.0, - "84": 1515676672.0, - "85": 1515676672.0, - "86": 1515676672.0, - "87": 1515676672.0, - "88": 1515676672.0, - "89": 1515676672.0, - "90": 1515676672.0, - "91": 1515676672.0, - "92": 1515676672.0, - "93": 1515676672.0, - "94": 1515676672.0, - "95": 1515676672.0, - "96": 1515676672.0, - "97": 1515676672.0, - "98": 1515676672.0, - "99": 1515676672.0, - "100": 1515676672.0 + "1": 971619840.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.29271, - "2": 0.42506, - "3": 0.68343, - "4": 0.36852, - "5": 0.35945, - "6": 0.70082, - "7": 0.36184, - "8": 0.36666, - "9": 0.36956, - "10": 0.36948, - "11": 0.34035, - "12": 0.33106, - "13": 0.32678, - "14": 0.50153, - "15": 0.32624, - "16": 0.32544, - "17": 0.33191, - "18": 0.32618, - "19": 0.3263, - "20": 0.33069, - "21": 0.32595, - "22": 0.3257, - "23": 0.33264, - "24": 0.32517, - "25": 0.32475, - "26": 0.33346, - "27": 0.33354, - "28": 0.32383, - "29": 0.33025, - "30": 0.32292, - "31": 0.32259, - "32": 0.33133, - "33": 0.32233, - "34": 0.32205, - "35": 0.32577, - "36": 0.33027, - "37": 0.32369, - "38": 0.3231, - "39": 0.32941, - "40": 0.32272, - "41": 0.32419, - "42": 0.32862, - "43": 0.32341, - "44": 0.32437, - "45": 0.3291, - "46": 0.32245, - "47": 0.32412, - "48": 0.32928, - "49": 0.32252, - "50": 0.3232, - "51": 0.3288, - "52": 0.32267, - "53": 0.32323, - "54": 0.33682, - "55": 0.32632, - "56": 0.32697, - "57": 0.33895, - "58": 0.32618, - "59": 0.32589, - "60": 0.3322, - "61": 0.3251, - "62": 0.32521, - "63": 0.33036, - "64": 0.32444, - "65": 0.32508, - "66": 0.33114, - "67": 0.32315, - "68": 0.32508, - "69": 0.3303, - "70": 0.32701, - "71": 0.32493, - "72": 0.32932, - "73": 0.32763, - "74": 0.32474, - "75": 0.32636, - "76": 0.33103, - "77": 0.32433, - "78": 0.32583, - "79": 0.33332, - "80": 0.32445, - "81": 0.32512, - "82": 0.33846, - "83": 0.32647, - "84": 0.32584, - "85": 0.33063, - "86": 0.32531, - "87": 0.32597, - "88": 0.33536, - "89": 0.32529, - "90": 0.32619, - "91": 0.33191, - "92": 0.32549, - "93": 0.32565, - "94": 0.33549, - "95": 0.32239, - "96": 0.32249, - "97": 0.32967, - "98": 0.3225, - "99": 0.32206, - "100": 0.32856 + "1": 5.18846, + "2": 0.36168, + "3": 0.29466, + "4": 0.29234, + "5": 0.29276, + "6": 0.29792, + "7": 0.29352, + "8": 0.2936, + "9": 0.29237, + "10": 0.29769, + "11": 0.29346, + "12": 0.29527, + "13": 0.29315, + "14": 0.29363, + "15": 0.29305, + "16": 0.29641, + "17": 0.29489, + "18": 0.29861, + "19": 0.29574, + "20": 0.29312, + "21": 0.29388, + "22": 0.29283, + "23": 0.29431, + "24": 0.29335, + "25": 0.29314, + "26": 0.29296, + "27": 0.29356, + "28": 0.29335, + "29": 0.29568, + "30": 0.29411, + "31": 0.29379, + "32": 0.29273, + "33": 0.29354, + "34": 0.29433, + "35": 0.29411, + "36": 0.29363, + "37": 0.2938, + "38": 0.29351, + "39": 0.29356, + "40": 0.29298, + "41": 0.29347, + "42": 0.29413, + "43": 0.29252, + "44": 0.29273, + "45": 0.29334, + "46": 0.29356, + "47": 0.29382, + "48": 0.29398, + "49": 0.2936, + "50": 0.29316, + "51": 0.29514, + "52": 0.28916, + "53": 0.29005, + "54": 0.28929, + "55": 0.28956, + "56": 0.28848, + "57": 0.28858, + "58": 0.28768, + "59": 0.28853, + "60": 0.29008, + "61": 0.2889, + "62": 0.28847, + "63": 0.28786, + "64": 0.28795, + "65": 0.28879, + "66": 0.28923, + "67": 0.28915, + "68": 0.28861, + "69": 0.28895, + "70": 0.28885, + "71": 0.28882, + "72": 0.28775, + "73": 0.28792, + "74": 0.28799, + "75": 0.28754, + "76": 0.28789, + "77": 0.2888, + "78": 0.28929, + "79": 0.28854, + "80": 0.28894, + "81": 0.28751, + "82": 0.28815, + "83": 0.2885, + "84": 0.28813, + "85": 0.28933, + "86": 0.28794, + "87": 0.28758, + "88": 0.28772, + "89": 0.28903, + "90": 0.28798, + "91": 0.28695, + "92": 0.28757, + "93": 0.28831, + "94": 0.28828, + "95": 0.28871, + "96": 0.28746, + "97": 0.28767, + "98": 0.28881, + "99": 0.2875, + "100": 0.28775 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json new file mode 100644 index 00000000000..d8c6189b1ed --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85966, + "2": 10.85779, + "3": 10.85806, + "4": 10.85111, + "5": 10.88314, + "6": 10.8873, + "7": 10.86205, + "8": 10.86979, + "9": 10.86489, + "10": 10.83833, + "11": 10.87881, + "12": 10.87319, + "13": 10.87924, + "14": 10.88927, + "15": 10.83002, + "16": 10.83231, + "17": 10.80059, + "18": 10.82546, + "19": 10.81132, + "20": 10.73125, + "21": 10.72292, + "22": 10.57559, + "23": 10.73211, + "24": 10.61836, + "25": 10.56602, + "26": 10.61211, + "27": 10.62224, + "28": 10.58011, + "29": 10.59905, + "30": 10.38311, + "31": 10.14181, + "32": 10.48118, + "33": 10.47322, + "34": 10.23556, + "35": 10.28738, + "36": 10.23493, + "37": 10.35785, + "38": 10.20631, + "39": 10.41397, + "40": 10.09217, + "41": 10.16577, + "42": 10.21856, + "43": 9.85974, + "44": 9.97115, + "45": 9.83404, + "46": 9.84483, + "47": 10.16033, + "48": 9.85135, + "49": 9.53781, + "50": 9.91577, + "51": 9.8567, + "52": 9.75178, + "53": 10.07652, + "54": 9.96084, + "55": 9.88221, + "56": 9.63206, + "57": 9.49147, + "58": 9.83408, + "59": 9.59352, + "60": 9.51388, + "61": 9.69802, + "62": 9.99154, + "63": 9.3723, + "64": 9.77839, + "65": 8.95518, + "66": 9.70976, + "67": 9.38198, + "68": 9.78701, + "69": 9.793, + "70": 9.73033, + "71": 9.61752, + "72": 9.58459, + "73": 9.48958, + "74": 8.94015, + "75": 9.43092, + "76": 9.09168, + "77": 10.06222, + "78": 9.72696, + "79": 9.37408, + "80": 9.40676, + "81": 9.47995, + "82": 9.69225, + "83": 9.31299, + "84": 9.41921, + "85": 9.61096, + "86": 9.06853, + "87": 9.59119, + "88": 9.74582, + "89": 9.60624, + "90": 9.81746, + "91": 9.34247, + "92": 9.35856, + "93": 9.07894, + "94": 8.82753, + "95": 9.51606, + "96": 9.52063, + "97": 9.31097, + "98": 9.67055, + "99": 8.88626, + "100": 9.40485 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1756.0, + "2": 2002.0, + "3": 1950.0, + "4": 1894.0, + "5": 2227.0, + "6": 2168.0, + "7": 2021.0, + "8": 1775.0, + "9": 2051.0, + "10": 1611.0, + "11": 2147.0, + "12": 1965.0, + "13": 2185.0, + "14": 2052.0, + "15": 2134.0, + "16": 2077.0, + "17": 2080.0, + "18": 1914.0, + "19": 1993.0, + "20": 1823.0, + "21": 2142.0, + "22": 1847.0, + "23": 2340.0, + "24": 1831.0, + "25": 1790.0, + "26": 1923.0, + "27": 2066.0, + "28": 2268.0, + "29": 2261.0, + "30": 2043.0, + "31": 1759.0, + "32": 2122.0, + "33": 2389.0, + "34": 2186.0, + "35": 2312.0, + "36": 2176.0, + "37": 2626.0, + "38": 2471.0, + "39": 2624.0, + "40": 2556.0, + "41": 2475.0, + "42": 2504.0, + "43": 2326.0, + "44": 2307.0, + "45": 2355.0, + "46": 2625.0, + "47": 2809.0, + "48": 2621.0, + "49": 2670.0, + "50": 2699.0, + "51": 2829.0, + "52": 2809.0, + "53": 3230.0, + "54": 2864.0, + "55": 2706.0, + "56": 2917.0, + "57": 2529.0, + "58": 3339.0, + "59": 3051.0, + "60": 2623.0, + "61": 3287.0, + "62": 2913.0, + "63": 2639.0, + "64": 3154.0, + "65": 2856.0, + "66": 3465.0, + "67": 2934.0, + "68": 2985.0, + "69": 3298.0, + "70": 3653.0, + "71": 3260.0, + "72": 2684.0, + "73": 3232.0, + "74": 2191.0, + "75": 2766.0, + "76": 3335.0, + "77": 3793.0, + "78": 3608.0, + "79": 3384.0, + "80": 3782.0, + "81": 3969.0, + "82": 3640.0, + "83": 3237.0, + "84": 3606.0, + "85": 3553.0, + "86": 3160.0, + "87": 4130.0, + "88": 3430.0, + "89": 3818.0, + "90": 3363.0, + "91": 3041.0, + "92": 3524.0, + "93": 3060.0, + "94": 3575.0, + "95": 3463.0, + "96": 3921.0, + "97": 3597.0, + "98": 4039.0, + "99": 3435.0, + "100": 3548.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 368387584.0, + "2": 368387584.0, + "3": 368387584.0, + "4": 368387584.0, + "5": 368387584.0, + "6": 368387584.0, + "7": 368387584.0, + "8": 368387584.0, + "9": 368387584.0, + "10": 368387584.0, + "11": 368387584.0, + "12": 368387584.0, + "13": 368387584.0, + "14": 368387584.0, + "15": 368387584.0, + "16": 368387584.0, + "17": 368387584.0, + "18": 368387584.0, + "19": 368387584.0, + "20": 368387584.0, + "21": 368387584.0, + "22": 368387584.0, + "23": 368387584.0, + "24": 368387584.0, + "25": 368387584.0, + "26": 368387584.0, + "27": 368387584.0, + "28": 368387584.0, + "29": 368387584.0, + "30": 368387584.0, + "31": 368387584.0, + "32": 368387584.0, + "33": 368387584.0, + "34": 368387584.0, + "35": 368387584.0, + "36": 368387584.0, + "37": 368387584.0, + "38": 368387584.0, + "39": 368387584.0, + "40": 368387584.0, + "41": 368387584.0, + "42": 368387584.0, + "43": 368387584.0, + "44": 368387584.0, + "45": 368387584.0, + "46": 368387584.0, + "47": 368387584.0, + "48": 368387584.0, + "49": 368387584.0, + "50": 368387584.0, + "51": 368387584.0, + "52": 368387584.0, + "53": 368387584.0, + "54": 368387584.0, + "55": 368387584.0, + "56": 368387584.0, + "57": 368387584.0, + "58": 368387584.0, + "59": 368387584.0, + "60": 368387584.0, + "61": 368387584.0, + "62": 368387584.0, + "63": 368387584.0, + "64": 368387584.0, + "65": 368387584.0, + "66": 368387584.0, + "67": 368387584.0, + "68": 368387584.0, + "69": 368387584.0, + "70": 368387584.0, + "71": 368387584.0, + "72": 368387584.0, + "73": 368387584.0, + "74": 368387584.0, + "75": 368387584.0, + "76": 368387584.0, + "77": 368387584.0, + "78": 368387584.0, + "79": 368387584.0, + "80": 368387584.0, + "81": 368387584.0, + "82": 368387584.0, + "83": 368387584.0, + "84": 368387584.0, + "85": 368387584.0, + "86": 368387584.0, + "87": 368387584.0, + "88": 368387584.0, + "89": 368387584.0, + "90": 368387584.0, + "91": 368387584.0, + "92": 368387584.0, + "93": 368387584.0, + "94": 368387584.0, + "95": 368387584.0, + "96": 368387584.0, + "97": 368387584.0, + "98": 368387584.0, + "99": 368387584.0, + "100": 368387584.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1030075904.0, + "2": 1161062912.0, + "3": 1161062912.0, + "4": 1161062912.0, + "5": 1161062912.0, + "6": 1161062912.0, + "7": 1161062912.0, + "8": 1161062912.0, + "9": 1161062912.0, + "10": 1161062912.0, + "11": 1161062912.0, + "12": 1161062912.0, + "13": 1161062912.0, + "14": 1161062912.0, + "15": 1161062912.0, + "16": 1161062912.0, + "17": 1161062912.0, + "18": 1161062912.0, + "19": 1161062912.0, + "20": 1161062912.0, + "21": 1161062912.0, + "22": 1161062912.0, + "23": 1161062912.0, + "24": 1161062912.0, + "25": 1161062912.0, + "26": 1161062912.0, + "27": 1161062912.0, + "28": 1161062912.0, + "29": 1161062912.0, + "30": 1161062912.0, + "31": 1161062912.0, + "32": 1161062912.0, + "33": 1161062912.0, + "34": 1161062912.0, + "35": 1161062912.0, + "36": 1161062912.0, + "37": 1161062912.0, + "38": 1161062912.0, + "39": 1161062912.0, + "40": 1161062912.0, + "41": 1161062912.0, + "42": 1161062912.0, + "43": 1161062912.0, + "44": 1161062912.0, + "45": 1161062912.0, + "46": 1161062912.0, + "47": 1161062912.0, + "48": 1161062912.0, + "49": 1161062912.0, + "50": 1161062912.0, + "51": 1161062912.0, + "52": 1161062912.0, + "53": 1161062912.0, + "54": 1161062912.0, + "55": 1161062912.0, + "56": 1161062912.0, + "57": 1161062912.0, + "58": 1161062912.0, + "59": 1161062912.0, + "60": 1161062912.0, + "61": 1161062912.0, + "62": 1161062912.0, + "63": 1161062912.0, + "64": 1161062912.0, + "65": 1161062912.0, + "66": 1161062912.0, + "67": 1161062912.0, + "68": 1161062912.0, + "69": 1161062912.0, + "70": 1161062912.0, + "71": 1161062912.0, + "72": 1161062912.0, + "73": 1161062912.0, + "74": 1161062912.0, + "75": 1161062912.0, + "76": 1161062912.0, + "77": 1161062912.0, + "78": 1161062912.0, + "79": 1161062912.0, + "80": 1161062912.0, + "81": 1161062912.0, + "82": 1161062912.0, + "83": 1161062912.0, + "84": 1161062912.0, + "85": 1161062912.0, + "86": 1161062912.0, + "87": 1161062912.0, + "88": 1161062912.0, + "89": 1161062912.0, + "90": 1161062912.0, + "91": 1161062912.0, + "92": 1161062912.0, + "93": 1161062912.0, + "94": 1161062912.0, + "95": 1161062912.0, + "96": 1161062912.0, + "97": 1161062912.0, + "98": 1161062912.0, + "99": 1161062912.0, + "100": 1161062912.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.69857, + "2": 0.5607, + "3": 0.4224, + "4": 0.408, + "5": 0.40601, + "6": 0.40622, + "7": 0.41117, + "8": 0.41277, + "9": 0.41135, + "10": 0.41075, + "11": 0.41036, + "12": 0.41422, + "13": 0.41066, + "14": 0.41137, + "15": 0.41203, + "16": 0.41967, + "17": 0.41058, + "18": 0.41138, + "19": 0.41123, + "20": 0.4101, + "21": 0.40646, + "22": 0.40698, + "23": 0.40933, + "24": 0.40762, + "25": 0.40817, + "26": 0.40815, + "27": 0.40808, + "28": 0.41113, + "29": 0.41068, + "30": 0.40954, + "31": 0.40816, + "32": 0.40856, + "33": 0.40858, + "34": 0.4094, + "35": 0.40789, + "36": 0.40721, + "37": 0.4076, + "38": 0.4074, + "39": 0.40856, + "40": 0.40702, + "41": 0.40749, + "42": 0.40775, + "43": 0.40697, + "44": 0.4055, + "45": 0.40519, + "46": 0.40481, + "47": 0.40587, + "48": 0.40507, + "49": 0.40669, + "50": 0.40562, + "51": 0.42093, + "52": 0.41028, + "53": 0.4088, + "54": 0.41137, + "55": 0.40983, + "56": 0.40893, + "57": 0.41252, + "58": 0.40982, + "59": 0.41034, + "60": 0.41176, + "61": 0.40849, + "62": 0.40923, + "63": 0.40913, + "64": 0.4084, + "65": 0.40845, + "66": 0.41123, + "67": 0.40774, + "68": 0.40592, + "69": 0.4048, + "70": 0.40668, + "71": 0.40553, + "72": 0.40771, + "73": 0.40622, + "74": 0.40899, + "75": 0.41092, + "76": 0.41085, + "77": 0.41068, + "78": 0.4084, + "79": 0.41086, + "80": 0.77469, + "81": 0.41176, + "82": 0.41234, + "83": 0.41225, + "84": 0.41217, + "85": 0.41295, + "86": 0.41174, + "87": 0.40921, + "88": 0.40831, + "89": 0.40924, + "90": 0.40844, + "91": 0.40982, + "92": 0.40955, + "93": 0.40963, + "94": 0.4085, + "95": 0.43734, + "96": 0.40945, + "97": 0.40886, + "98": 0.41034, + "99": 0.40924, + "100": 0.40948 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json index b9747029488..1091699bf9a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86209, "5": 10.88253, "10": 10.83642, "15": 10.82909, "20": 10.73237, "25": 10.56392, "30": 10.38484, "35": 10.28951, "40": 10.09133, "45": 9.83487, "50": 9.91604, "55": 9.88232, "60": 9.51376, "65": 8.95392, "70": 9.73094, "75": 9.4313, "80": 9.40597, "85": 9.61137, "90": 9.81743, "95": 9.51571, "100": 9.40429}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1780.0, "5": 2137.0, "10": 1588.0, "15": 2083.0, "20": 1816.0, "25": 1862.0, "30": 2045.0, "35": 2205.0, "40": 2629.0, "45": 2441.0, "50": 2734.0, "55": 2662.0, "60": 2726.0, "65": 3011.0, "70": 3737.0, "75": 2771.0, "80": 3804.0, "85": 3588.0, "90": 3384.0, "95": 3590.0, "100": 3454.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 368387072.0, "5": 368387072.0, "10": 368387072.0, "15": 368387072.0, "20": 368387072.0, "25": 368387072.0, "30": 368387072.0, "35": 368387072.0, "40": 368387072.0, "45": 368387072.0, "50": 368387072.0, "55": 368387072.0, "60": 368387072.0, "65": 368387072.0, "70": 368387072.0, "75": 368387072.0, "80": 368387072.0, "85": 368387072.0, "90": 368387072.0, "95": 368387072.0, "100": 368387072.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1510972416.0, "5": 1645104640.0, "10": 1645105152.0, "15": 1645105152.0, "20": 1645105152.0, "25": 1645105152.0, "30": 1645105152.0, "35": 1645105152.0, "40": 1645105152.0, "45": 1645105152.0, "50": 1645105152.0, "55": 1645105152.0, "60": 1645105152.0, "65": 1645105152.0, "70": 1645105152.0, "75": 1647199744.0, "80": 1647199744.0, "85": 1647199744.0, "90": 1647199744.0, "95": 1647199744.0, "100": 1647199744.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 9.70786, "5": 0.41822, "10": 0.42132, "15": 0.41057, "20": 0.4105, "25": 0.41988, "30": 0.41035, "35": 0.41599, "40": 0.40546, "45": 0.40496, "50": 0.40588, "55": 0.40982, "60": 0.4157, "65": 0.40455, "70": 0.40547, "75": 0.41128, "80": 0.41661, "85": 0.42011, "90": 0.41367, "95": 0.41851, "100": 0.40235}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85966, + "2": 10.85779, + "3": 10.85806, + "4": 10.85111, + "5": 10.88314, + "6": 10.8873, + "7": 10.86205, + "8": 10.86979, + "9": 10.86489, + "10": 10.83833, + "11": 10.87881, + "12": 10.87319, + "13": 10.87924, + "14": 10.88927, + "15": 10.83002, + "16": 10.83231, + "17": 10.80059, + "18": 10.82546, + "19": 10.81132, + "20": 10.73125, + "21": 10.72292, + "22": 10.57559, + "23": 10.73211, + "24": 10.61836, + "25": 10.56602, + "26": 10.61211, + "27": 10.62224, + "28": 10.58011, + "29": 10.59905, + "30": 10.38311, + "31": 10.14181, + "32": 10.48118, + "33": 10.47322, + "34": 10.23556, + "35": 10.28738, + "36": 10.23493, + "37": 10.35785, + "38": 10.20631, + "39": 10.41397, + "40": 10.09217, + "41": 10.16577, + "42": 10.21856, + "43": 9.85974, + "44": 9.97115, + "45": 9.83404, + "46": 9.84483, + "47": 10.16033, + "48": 9.85135, + "49": 9.53781, + "50": 9.91577, + "51": 9.8567, + "52": 9.75178, + "53": 10.07652, + "54": 9.96084, + "55": 9.88221, + "56": 9.63206, + "57": 9.49147, + "58": 9.83408, + "59": 9.59352, + "60": 9.51388, + "61": 9.69802, + "62": 9.99154, + "63": 9.3723, + "64": 9.77839, + "65": 8.95518, + "66": 9.70976, + "67": 9.38198, + "68": 9.78701, + "69": 9.793, + "70": 9.73033, + "71": 9.61752, + "72": 9.58459, + "73": 9.48958, + "74": 8.94015, + "75": 9.43092, + "76": 9.09168, + "77": 10.06222, + "78": 9.72696, + "79": 9.37408, + "80": 9.40676, + "81": 9.47995, + "82": 9.69225, + "83": 9.31299, + "84": 9.41921, + "85": 9.61096, + "86": 9.06853, + "87": 9.59119, + "88": 9.74582, + "89": 9.60624, + "90": 9.81746, + "91": 9.34247, + "92": 9.35856, + "93": 9.07894, + "94": 8.82753, + "95": 9.51606, + "96": 9.52063, + "97": 9.31097, + "98": 9.67055, + "99": 8.88626, + "100": 9.40485 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1756.0, + "2": 2002.0, + "3": 1950.0, + "4": 1894.0, + "5": 2227.0, + "6": 2168.0, + "7": 2021.0, + "8": 1775.0, + "9": 2051.0, + "10": 1611.0, + "11": 2147.0, + "12": 1965.0, + "13": 2185.0, + "14": 2052.0, + "15": 2134.0, + "16": 2077.0, + "17": 2080.0, + "18": 1914.0, + "19": 1993.0, + "20": 1823.0, + "21": 2142.0, + "22": 1847.0, + "23": 2340.0, + "24": 1831.0, + "25": 1790.0, + "26": 1923.0, + "27": 2066.0, + "28": 2268.0, + "29": 2261.0, + "30": 2043.0, + "31": 1759.0, + "32": 2122.0, + "33": 2389.0, + "34": 2186.0, + "35": 2312.0, + "36": 2176.0, + "37": 2626.0, + "38": 2471.0, + "39": 2624.0, + "40": 2556.0, + "41": 2475.0, + "42": 2504.0, + "43": 2326.0, + "44": 2307.0, + "45": 2355.0, + "46": 2625.0, + "47": 2809.0, + "48": 2621.0, + "49": 2670.0, + "50": 2699.0, + "51": 2829.0, + "52": 2809.0, + "53": 3230.0, + "54": 2864.0, + "55": 2706.0, + "56": 2917.0, + "57": 2529.0, + "58": 3339.0, + "59": 3051.0, + "60": 2623.0, + "61": 3287.0, + "62": 2913.0, + "63": 2639.0, + "64": 3154.0, + "65": 2856.0, + "66": 3465.0, + "67": 2934.0, + "68": 2985.0, + "69": 3298.0, + "70": 3653.0, + "71": 3260.0, + "72": 2684.0, + "73": 3232.0, + "74": 2191.0, + "75": 2766.0, + "76": 3335.0, + "77": 3793.0, + "78": 3608.0, + "79": 3384.0, + "80": 3782.0, + "81": 3969.0, + "82": 3640.0, + "83": 3237.0, + "84": 3606.0, + "85": 3553.0, + "86": 3160.0, + "87": 4130.0, + "88": 3430.0, + "89": 3818.0, + "90": 3363.0, + "91": 3041.0, + "92": 3524.0, + "93": 3060.0, + "94": 3575.0, + "95": 3463.0, + "96": 3921.0, + "97": 3597.0, + "98": 4039.0, + "99": 3435.0, + "100": 3548.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 368387584.0, + "2": 368387584.0, + "3": 368387584.0, + "4": 368387584.0, + "5": 368387584.0, + "6": 368387584.0, + "7": 368387584.0, + "8": 368387584.0, + "9": 368387584.0, + "10": 368387584.0, + "11": 368387584.0, + "12": 368387584.0, + "13": 368387584.0, + "14": 368387584.0, + "15": 368387584.0, + "16": 368387584.0, + "17": 368387584.0, + "18": 368387584.0, + "19": 368387584.0, + "20": 368387584.0, + "21": 368387584.0, + "22": 368387584.0, + "23": 368387584.0, + "24": 368387584.0, + "25": 368387584.0, + "26": 368387584.0, + "27": 368387584.0, + "28": 368387584.0, + "29": 368387584.0, + "30": 368387584.0, + "31": 368387584.0, + "32": 368387584.0, + "33": 368387584.0, + "34": 368387584.0, + "35": 368387584.0, + "36": 368387584.0, + "37": 368387584.0, + "38": 368387584.0, + "39": 368387584.0, + "40": 368387584.0, + "41": 368387584.0, + "42": 368387584.0, + "43": 368387584.0, + "44": 368387584.0, + "45": 368387584.0, + "46": 368387584.0, + "47": 368387584.0, + "48": 368387584.0, + "49": 368387584.0, + "50": 368387584.0, + "51": 368387584.0, + "52": 368387584.0, + "53": 368387584.0, + "54": 368387584.0, + "55": 368387584.0, + "56": 368387584.0, + "57": 368387584.0, + "58": 368387584.0, + "59": 368387584.0, + "60": 368387584.0, + "61": 368387584.0, + "62": 368387584.0, + "63": 368387584.0, + "64": 368387584.0, + "65": 368387584.0, + "66": 368387584.0, + "67": 368387584.0, + "68": 368387584.0, + "69": 368387584.0, + "70": 368387584.0, + "71": 368387584.0, + "72": 368387584.0, + "73": 368387584.0, + "74": 368387584.0, + "75": 368387584.0, + "76": 368387584.0, + "77": 368387584.0, + "78": 368387584.0, + "79": 368387584.0, + "80": 368387584.0, + "81": 368387584.0, + "82": 368387584.0, + "83": 368387584.0, + "84": 368387584.0, + "85": 368387584.0, + "86": 368387584.0, + "87": 368387584.0, + "88": 368387584.0, + "89": 368387584.0, + "90": 368387584.0, + "91": 368387584.0, + "92": 368387584.0, + "93": 368387584.0, + "94": 368387584.0, + "95": 368387584.0, + "96": 368387584.0, + "97": 368387584.0, + "98": 368387584.0, + "99": 368387584.0, + "100": 368387584.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1030075904.0, + "2": 1161062912.0, + "3": 1161062912.0, + "4": 1161062912.0, + "5": 1161062912.0, + "6": 1161062912.0, + "7": 1161062912.0, + "8": 1161062912.0, + "9": 1161062912.0, + "10": 1161062912.0, + "11": 1161062912.0, + "12": 1161062912.0, + "13": 1161062912.0, + "14": 1161062912.0, + "15": 1161062912.0, + "16": 1161062912.0, + "17": 1161062912.0, + "18": 1161062912.0, + "19": 1161062912.0, + "20": 1161062912.0, + "21": 1161062912.0, + "22": 1161062912.0, + "23": 1161062912.0, + "24": 1161062912.0, + "25": 1161062912.0, + "26": 1161062912.0, + "27": 1161062912.0, + "28": 1161062912.0, + "29": 1161062912.0, + "30": 1161062912.0, + "31": 1161062912.0, + "32": 1161062912.0, + "33": 1161062912.0, + "34": 1161062912.0, + "35": 1161062912.0, + "36": 1161062912.0, + "37": 1161062912.0, + "38": 1161062912.0, + "39": 1161062912.0, + "40": 1161062912.0, + "41": 1161062912.0, + "42": 1161062912.0, + "43": 1161062912.0, + "44": 1161062912.0, + "45": 1161062912.0, + "46": 1161062912.0, + "47": 1161062912.0, + "48": 1161062912.0, + "49": 1161062912.0, + "50": 1161062912.0, + "51": 1161062912.0, + "52": 1161062912.0, + "53": 1161062912.0, + "54": 1161062912.0, + "55": 1161062912.0, + "56": 1161062912.0, + "57": 1161062912.0, + "58": 1161062912.0, + "59": 1161062912.0, + "60": 1161062912.0, + "61": 1161062912.0, + "62": 1161062912.0, + "63": 1161062912.0, + "64": 1161062912.0, + "65": 1161062912.0, + "66": 1161062912.0, + "67": 1161062912.0, + "68": 1161062912.0, + "69": 1161062912.0, + "70": 1161062912.0, + "71": 1161062912.0, + "72": 1161062912.0, + "73": 1161062912.0, + "74": 1161062912.0, + "75": 1161062912.0, + "76": 1161062912.0, + "77": 1161062912.0, + "78": 1161062912.0, + "79": 1161062912.0, + "80": 1161062912.0, + "81": 1161062912.0, + "82": 1161062912.0, + "83": 1161062912.0, + "84": 1161062912.0, + "85": 1161062912.0, + "86": 1161062912.0, + "87": 1161062912.0, + "88": 1161062912.0, + "89": 1161062912.0, + "90": 1161062912.0, + "91": 1161062912.0, + "92": 1161062912.0, + "93": 1161062912.0, + "94": 1161062912.0, + "95": 1161062912.0, + "96": 1161062912.0, + "97": 1161062912.0, + "98": 1161062912.0, + "99": 1161062912.0, + "100": 1161062912.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.72553, + "2": 0.52446, + "3": 0.41527, + "4": 0.41699, + "5": 0.41496, + "6": 0.41411, + "7": 0.41829, + "8": 0.41655, + "9": 0.41643, + "10": 0.42008, + "11": 0.41959, + "12": 0.41842, + "13": 0.41485, + "14": 0.41643, + "15": 0.41486, + "16": 0.41617, + "17": 0.41476, + "18": 0.42598, + "19": 0.41821, + "20": 0.41457, + "21": 0.41579, + "22": 0.41438, + "23": 0.41644, + "24": 0.41499, + "25": 0.41537, + "26": 0.41593, + "27": 0.42875, + "28": 0.41636, + "29": 0.41505, + "30": 0.4148, + "31": 0.41806, + "32": 0.41549, + "33": 0.41482, + "34": 0.41559, + "35": 0.4156, + "36": 0.4152, + "37": 0.4152, + "38": 0.4154, + "39": 0.41674, + "40": 0.41745, + "41": 0.41582, + "42": 0.41548, + "43": 0.41428, + "44": 0.4158, + "45": 0.41469, + "46": 0.41584, + "47": 0.41662, + "48": 0.41588, + "49": 0.41548, + "50": 0.42504, + "51": 0.41857, + "52": 0.40985, + "53": 0.40877, + "54": 0.41013, + "55": 0.40869, + "56": 0.84381, + "57": 0.41437, + "58": 0.42502, + "59": 0.41122, + "60": 0.41956, + "61": 0.40996, + "62": 0.40983, + "63": 0.41144, + "64": 0.41126, + "65": 0.41361, + "66": 0.41243, + "67": 0.41431, + "68": 0.4396, + "69": 0.42434, + "70": 0.41269, + "71": 0.42108, + "72": 0.41357, + "73": 0.41116, + "74": 0.41086, + "75": 0.41041, + "76": 0.41106, + "77": 0.41, + "78": 0.41669, + "79": 0.41627, + "80": 0.41237, + "81": 0.41157, + "82": 0.41168, + "83": 0.41229, + "84": 0.41209, + "85": 0.41258, + "86": 0.41294, + "87": 0.41185, + "88": 0.41106, + "89": 0.41159, + "90": 0.41277, + "91": 0.41162, + "92": 0.41309, + "93": 0.41351, + "94": 0.40941, + "95": 0.40961, + "96": 0.41012, + "97": 0.40887, + "98": 0.40809, + "99": 0.40865, + "100": 0.40854 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..128bc5b55bb --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84082, + "2": 10.83544, + "3": 10.83361, + "4": 10.8202, + "5": 10.84212, + "6": 10.87215, + "7": 10.83489, + "8": 10.8434, + "9": 10.84794, + "10": 10.81681, + "11": 10.86063, + "12": 10.85821, + "13": 10.87957, + "14": 10.87773, + "15": 10.85163, + "16": 10.8472, + "17": 10.84071, + "18": 10.86582, + "19": 10.85937, + "20": 10.85131, + "21": 10.85552, + "22": 10.82535, + "23": 10.86895, + "24": 10.82877, + "25": 10.81945, + "26": 10.83385, + "27": 10.82583, + "28": 10.84435, + "29": 10.83124, + "30": 10.75388, + "31": 10.66502, + "32": 10.78819, + "33": 10.76332, + "34": 10.67654, + "35": 10.68319, + "36": 10.63342, + "37": 10.68482, + "38": 10.60112, + "39": 10.69463, + "40": 10.52433, + "41": 10.54208, + "42": 10.56578, + "43": 10.34551, + "44": 10.38873, + "45": 10.31101, + "46": 10.30065, + "47": 10.48062, + "48": 10.28105, + "49": 10.05975, + "50": 10.29413, + "51": 10.23775, + "52": 10.15443, + "53": 10.36085, + "54": 10.26927, + "55": 10.2161, + "56": 9.99594, + "57": 9.8744, + "58": 10.14007, + "59": 9.93447, + "60": 9.84864, + "61": 9.98549, + "62": 10.2164, + "63": 9.69034, + "64": 10.0182, + "65": 9.30046, + "66": 9.9355, + "67": 9.63051, + "68": 9.99128, + "69": 9.9852, + "70": 9.92463, + "71": 9.81436, + "72": 9.79481, + "73": 9.68082, + "74": 9.1945, + "75": 9.60407, + "76": 9.28537, + "77": 10.18507, + "78": 9.86718, + "79": 9.52407, + "80": 9.55749, + "81": 9.62863, + "82": 9.81568, + "83": 9.45708, + "84": 9.53654, + "85": 9.73266, + "86": 9.20138, + "87": 9.69524, + "88": 9.85412, + "89": 9.71648, + "90": 9.91047, + "91": 9.45992, + "92": 9.46603, + "93": 9.19321, + "94": 8.94, + "95": 9.60607, + "96": 9.62214, + "97": 9.39796, + "98": 9.76023, + "99": 8.99097, + "100": 9.49505 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 454770688.0, + "2": 454770688.0, + "3": 454770688.0, + "4": 454770688.0, + "5": 454770688.0, + "6": 454770688.0, + "7": 454770688.0, + "8": 454770688.0, + "9": 454770688.0, + "10": 454770688.0, + "11": 454770688.0, + "12": 454770688.0, + "13": 454770688.0, + "14": 454770688.0, + "15": 454770688.0, + "16": 454770688.0, + "17": 454770688.0, + "18": 518880768.0, + "19": 518880768.0, + "20": 518880768.0, + "21": 518880768.0, + "22": 518880768.0, + "23": 518880768.0, + "24": 518880768.0, + "25": 518880768.0, + "26": 518880768.0, + "27": 518880768.0, + "28": 518880768.0, + "29": 518880768.0, + "30": 518880768.0, + "31": 518880768.0, + "32": 518880768.0, + "33": 518880768.0, + "34": 518880768.0, + "35": 518880768.0, + "36": 518880768.0, + "37": 518880768.0, + "38": 518880768.0, + "39": 518880768.0, + "40": 518880768.0, + "41": 518880768.0, + "42": 518880768.0, + "43": 518880768.0, + "44": 518880768.0, + "45": 518880768.0, + "46": 518880768.0, + "47": 518880768.0, + "48": 518880768.0, + "49": 518880768.0, + "50": 518880768.0, + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, + "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, + "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, + "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, + "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, + "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, + "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, + "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, + "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, + "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, + "100": 518880768.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4511150592.0, + "2": 4544705536.0, + "3": 4544705536.0, + "4": 4544705536.0, + "5": 4544705536.0, + "6": 4544705536.0, + "7": 4544705536.0, + "8": 4544705536.0, + "9": 4544705536.0, + "10": 4544705536.0, + "11": 4544705536.0, + "12": 4544705536.0, + "13": 4544705536.0, + "14": 4544705536.0, + "15": 4544705536.0, + "16": 4544705536.0, + "17": 4544705536.0, + "18": 4544705536.0, + "19": 4607767040.0, + "20": 4607767040.0, + "21": 4607767040.0, + "22": 4607767040.0, + "23": 4607767040.0, + "24": 4607767040.0, + "25": 4607767040.0, + "26": 4607767040.0, + "27": 4607767040.0, + "28": 4607767040.0, + "29": 4607767040.0, + "30": 4607767040.0, + "31": 4607767040.0, + "32": 4607767040.0, + "33": 4607767040.0, + "34": 4607767040.0, + "35": 4607767040.0, + "36": 4607767040.0, + "37": 4607767040.0, + "38": 4607767040.0, + "39": 4607767040.0, + "40": 4607767040.0, + "41": 4607767040.0, + "42": 4607767040.0, + "43": 4607767040.0, + "44": 4607767040.0, + "45": 4607767040.0, + "46": 4607767040.0, + "47": 4607767040.0, + "48": 4607767040.0, + "49": 4607767040.0, + "50": 4607767040.0, + "51": 4607767040.0, + "52": 4607767040.0, + "53": 4607767040.0, + "54": 4607767040.0, + "55": 4607767040.0, + "56": 4607767040.0, + "57": 4607767040.0, + "58": 4607767040.0, + "59": 4607767040.0, + "60": 4607767040.0, + "61": 4607767040.0, + "62": 4607767040.0, + "63": 4607767040.0, + "64": 4607767040.0, + "65": 4607767040.0, + "66": 4607767040.0, + "67": 4607767040.0, + "68": 4607767040.0, + "69": 4607767040.0, + "70": 4607767040.0, + "71": 4607767040.0, + "72": 4607767040.0, + "73": 4607767040.0, + "74": 4607767040.0, + "75": 4607767040.0, + "76": 4607767040.0, + "77": 4607767040.0, + "78": 4607767040.0, + "79": 4607767040.0, + "80": 4607767040.0, + "81": 4607767040.0, + "82": 4607767040.0, + "83": 4607767040.0, + "84": 4607767040.0, + "85": 4607767040.0, + "86": 4607767040.0, + "87": 4607767040.0, + "88": 4607767040.0, + "89": 4607767040.0, + "90": 4607767040.0, + "91": 4607767040.0, + "92": 4607767040.0, + "93": 4607767040.0, + "94": 4607767040.0, + "95": 4607767040.0, + "96": 4607767040.0, + "97": 4607767040.0, + "98": 4607767040.0, + "99": 4607767040.0, + "100": 4607767040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.20459, + "2": 0.14512, + "3": 0.09601, + "4": 0.09045, + "5": 0.09053, + "6": 0.08966, + "7": 0.08974, + "8": 0.08915, + "9": 0.08902, + "10": 0.08919, + "11": 0.08955, + "12": 0.08907, + "13": 0.08947, + "14": 0.08882, + "15": 0.08864, + "16": 0.08897, + "17": 0.08902, + "18": 0.14106, + "19": 0.0977, + "20": 0.09649, + "21": 0.09724, + "22": 0.09168, + "23": 0.09714, + "24": 0.09658, + "25": 0.09708, + "26": 0.09693, + "27": 0.09615, + "28": 0.09697, + "29": 0.09661, + "30": 0.09679, + "31": 0.09716, + "32": 0.09671, + "33": 0.09697, + "34": 0.09691, + "35": 0.09689, + "36": 0.09592, + "37": 0.09711, + "38": 0.09635, + "39": 0.0969, + "40": 0.09642, + "41": 0.09689, + "42": 0.09616, + "43": 0.0971, + "44": 0.09592, + "45": 0.09674, + "46": 0.09586, + "47": 0.09669, + "48": 0.096, + "49": 0.09678, + "50": 0.09589, + "51": 0.09963, + "52": 0.09882, + "53": 0.09639, + "54": 0.09745, + "55": 0.09701, + "56": 0.0971, + "57": 0.0929, + "58": 0.09316, + "59": 0.09353, + "60": 0.0935, + "61": 0.0935, + "62": 0.09059, + "63": 0.09063, + "64": 0.09203, + "65": 0.09133, + "66": 0.09044, + "67": 0.09102, + "68": 0.08852, + "69": 0.08696, + "70": 0.08735, + "71": 0.08724, + "72": 0.08697, + "73": 0.08755, + "74": 0.08731, + "75": 0.08922, + "76": 0.08886, + "77": 0.09297, + "78": 0.0878, + "79": 0.08778, + "80": 0.08751, + "81": 0.08754, + "82": 0.08716, + "83": 0.08674, + "84": 0.0865, + "85": 0.08602, + "86": 0.08613, + "87": 0.08623, + "88": 0.08639, + "89": 0.08691, + "90": 0.08616, + "91": 0.08641, + "92": 0.08645, + "93": 0.08643, + "94": 0.08632, + "95": 0.0862, + "96": 0.08615, + "97": 0.08618, + "98": 0.0866, + "99": 0.0867, + "100": 0.0861 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": 1208.0, + "19": 1541.0, + "20": 1173.0, + "21": 1368.0, + "22": "nan", + "23": 1317.0, + "24": 1131.0, + "25": 1119.0, + "26": 1067.0, + "27": 1300.0, + "28": 1549.0, + "29": 1501.0, + "30": 1354.0, + "31": 1044.0, + "32": 1202.0, + "33": 1503.0, + "34": 1276.0, + "35": 1207.0, + "36": 1217.0, + "37": 1430.0, + "38": 1473.0, + "39": 1525.0, + "40": 1413.0, + "41": 1441.0, + "42": 1487.0, + "43": 1151.0, + "44": 1217.0, + "45": 1314.0, + "46": 1265.0, + "47": 1868.0, + "48": 1314.0, + "49": 1105.0, + "50": 1476.0, + "51": 1508.0, + "52": 1400.0, + "53": 1740.0, + "54": 1498.0, + "55": 1551.0, + "56": 1363.0, + "57": 1465.0, + "58": 1610.0, + "59": 1574.0, + "60": 1599.0, + "61": 1727.0, + "62": 1804.0, + "63": 1590.0, + "64": 1813.0, + "65": 1398.0, + "66": 1738.0, + "67": 1536.0, + "68": 1764.0, + "69": 1781.0, + "70": 1926.0, + "71": 1950.0, + "72": 1461.0, + "73": 1985.0, + "74": 1345.0, + "75": 1871.0, + "76": 1732.0, + "77": 2086.0, + "78": 2075.0, + "79": 1992.0, + "80": 2260.0, + "81": 2300.0, + "82": 2290.0, + "83": 1774.0, + "84": 2172.0, + "85": 2216.0, + "86": 2038.0, + "87": 2741.0, + "88": 2079.0, + "89": 2349.0, + "90": 2315.0, + "91": 1875.0, + "92": 2611.0, + "93": 2053.0, + "94": 2220.0, + "95": 2296.0, + "96": 2665.0, + "97": 2516.0, + "98": 2549.0, + "99": 2378.0, + "100": 2257.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index a6254c12055..8d29fc96a7f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.84012, - "5": 10.84147, - "10": 10.82107, - "15": 10.85423, - "20": 10.85111, - "25": 10.82229, - "30": 10.75584, - "35": 10.68389, - "40": 10.52756, - "45": 10.3119, - "50": 10.29392, - "55": 10.21882, - "60": 9.8477, - "65": 9.30027, - "70": 9.92473, - "75": 9.60615, - "80": 9.55867, - "85": 9.7331, - "90": 9.91024, - "95": 9.60586, - "100": 9.49453 + "1": 10.84082, + "2": 10.83544, + "3": 10.83361, + "4": 10.8202, + "5": 10.84212, + "6": 10.87215, + "7": 10.83489, + "8": 10.8434, + "9": 10.84794, + "10": 10.81681, + "11": 10.86063, + "12": 10.85821, + "13": 10.87957, + "14": 10.87773, + "15": 10.85163, + "16": 10.8472, + "17": 10.84071, + "18": 10.86582, + "19": 10.85937, + "20": 10.85131, + "21": 10.85552, + "22": 10.82535, + "23": 10.86895, + "24": 10.82877, + "25": 10.81945, + "26": 10.83385, + "27": 10.82583, + "28": 10.84435, + "29": 10.83124, + "30": 10.75388, + "31": 10.66502, + "32": 10.78819, + "33": 10.76332, + "34": 10.67654, + "35": 10.68319, + "36": 10.63342, + "37": 10.68482, + "38": 10.60112, + "39": 10.69463, + "40": 10.52433, + "41": 10.54208, + "42": 10.56578, + "43": 10.34551, + "44": 10.38873, + "45": 10.31101, + "46": 10.30065, + "47": 10.48062, + "48": 10.28105, + "49": 10.05975, + "50": 10.29413, + "51": 10.23775, + "52": 10.15443, + "53": 10.36085, + "54": 10.26927, + "55": 10.2161, + "56": 9.99594, + "57": 9.8744, + "58": 10.14007, + "59": 9.93447, + "60": 9.84864, + "61": 9.98549, + "62": 10.2164, + "63": 9.69034, + "64": 10.0182, + "65": 9.30046, + "66": 9.9355, + "67": 9.63051, + "68": 9.99128, + "69": 9.9852, + "70": 9.92463, + "71": 9.81436, + "72": 9.79481, + "73": 9.68082, + "74": 9.1945, + "75": 9.60407, + "76": 9.28537, + "77": 10.18507, + "78": 9.86718, + "79": 9.52407, + "80": 9.55749, + "81": 9.62863, + "82": 9.81568, + "83": 9.45708, + "84": 9.53654, + "85": 9.73266, + "86": 9.20138, + "87": 9.69524, + "88": 9.85412, + "89": 9.71648, + "90": 9.91047, + "91": 9.45992, + "92": 9.46603, + "93": 9.19321, + "94": 8.94, + "95": 9.60607, + "96": 9.62214, + "97": 9.39796, + "98": 9.76023, + "99": 8.99097, + "100": 9.49505 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 454770688.0, + "2": 454770688.0, + "3": 454770688.0, + "4": 454770688.0, "5": 454770688.0, + "6": 454770688.0, + "7": 454770688.0, + "8": 454770688.0, + "9": 454770688.0, "10": 454770688.0, + "11": 454770688.0, + "12": 454770688.0, + "13": 454770688.0, + "14": 454770688.0, "15": 454770688.0, + "16": 454770688.0, + "17": 454770688.0, + "18": 518880768.0, + "19": 518880768.0, "20": 518880768.0, + "21": 518880768.0, + "22": 518880768.0, + "23": 518880768.0, + "24": 518880768.0, "25": 518880768.0, + "26": 518880768.0, + "27": 518880768.0, + "28": 518880768.0, + "29": 518880768.0, "30": 518880768.0, + "31": 518880768.0, + "32": 518880768.0, + "33": 518880768.0, + "34": 518880768.0, "35": 518880768.0, + "36": 518880768.0, + "37": 518880768.0, + "38": 518880768.0, + "39": 518880768.0, "40": 518880768.0, + "41": 518880768.0, + "42": 518880768.0, + "43": 518880768.0, + "44": 518880768.0, "45": 518880768.0, + "46": 518880768.0, + "47": 518880768.0, + "48": 518880768.0, + "49": 518880768.0, "50": 518880768.0, + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, "100": 518880768.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 4511150592.0, + "2": 4544705536.0, + "3": 4544705536.0, + "4": 4544705536.0, "5": 4544705536.0, + "6": 4544705536.0, + "7": 4544705536.0, + "8": 4544705536.0, + "9": 4544705536.0, "10": 4544705536.0, + "11": 4544705536.0, + "12": 4544705536.0, + "13": 4544705536.0, + "14": 4544705536.0, "15": 4544705536.0, + "16": 4544705536.0, + "17": 4544705536.0, + "18": 4544705536.0, + "19": 4607767040.0, "20": 4607767040.0, + "21": 4607767040.0, + "22": 4607767040.0, + "23": 4607767040.0, + "24": 4607767040.0, "25": 4607767040.0, + "26": 4607767040.0, + "27": 4607767040.0, + "28": 4607767040.0, + "29": 4607767040.0, "30": 4607767040.0, + "31": 4607767040.0, + "32": 4607767040.0, + "33": 4607767040.0, + "34": 4607767040.0, "35": 4607767040.0, + "36": 4607767040.0, + "37": 4607767040.0, + "38": 4607767040.0, + "39": 4607767040.0, "40": 4607767040.0, + "41": 4607767040.0, + "42": 4607767040.0, + "43": 4607767040.0, + "44": 4607767040.0, "45": 4607767040.0, + "46": 4607767040.0, + "47": 4607767040.0, + "48": 4607767040.0, + "49": 4607767040.0, "50": 4607767040.0, + "51": 4607767040.0, + "52": 4607767040.0, + "53": 4607767040.0, + "54": 4607767040.0, "55": 4607767040.0, + "56": 4607767040.0, + "57": 4607767040.0, + "58": 4607767040.0, + "59": 4607767040.0, "60": 4607767040.0, + "61": 4607767040.0, + "62": 4607767040.0, + "63": 4607767040.0, + "64": 4607767040.0, "65": 4607767040.0, + "66": 4607767040.0, + "67": 4607767040.0, + "68": 4607767040.0, + "69": 4607767040.0, "70": 4607767040.0, + "71": 4607767040.0, + "72": 4607767040.0, + "73": 4607767040.0, + "74": 4607767040.0, "75": 4607767040.0, + "76": 4607767040.0, + "77": 4607767040.0, + "78": 4607767040.0, + "79": 4607767040.0, "80": 4607767040.0, + "81": 4607767040.0, + "82": 4607767040.0, + "83": 4607767040.0, + "84": 4607767040.0, "85": 4607767040.0, + "86": 4607767040.0, + "87": 4607767040.0, + "88": 4607767040.0, + "89": 4607767040.0, "90": 4607767040.0, + "91": 4607767040.0, + "92": 4607767040.0, + "93": 4607767040.0, + "94": 4607767040.0, "95": 4607767040.0, + "96": 4607767040.0, + "97": 4607767040.0, + "98": 4607767040.0, + "99": 4607767040.0, "100": 4607767040.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.36364, - "5": 0.11207, - "10": 0.09807, - "15": 0.11189, - "20": 0.12039, - "25": 0.10422, - "30": 0.10453, - "35": 0.10312, - "40": 0.10542, - "45": 0.10362, - "50": 0.10448, - "55": 0.10212, - "60": 0.10194, - "65": 0.10259, - "70": 0.10183, - "75": 0.10119, - "80": 0.10042, - "85": 0.10084, - "90": 0.10028, - "95": 0.10004, - "100": 0.09998 + "1": 4.2592, + "2": 0.13544, + "3": 0.09999, + "4": 0.08273, + "5": 0.08157, + "6": 0.08266, + "7": 0.08111, + "8": 0.08184, + "9": 0.08109, + "10": 0.08281, + "11": 0.08041, + "12": 0.08186, + "13": 0.08098, + "14": 0.08513, + "15": 0.0821, + "16": 0.08144, + "17": 0.08052, + "18": 0.13091, + "19": 0.08819, + "20": 0.08804, + "21": 0.08818, + "22": 0.08404, + "23": 0.08729, + "24": 0.08805, + "25": 0.08736, + "26": 0.08811, + "27": 0.08757, + "28": 0.08887, + "29": 0.08961, + "30": 0.0883, + "31": 0.08788, + "32": 0.08884, + "33": 0.08833, + "34": 0.08811, + "35": 0.08831, + "36": 0.08859, + "37": 0.08809, + "38": 0.08879, + "39": 0.08769, + "40": 0.0883, + "41": 0.08757, + "42": 0.08797, + "43": 0.08669, + "44": 0.08751, + "45": 0.08893, + "46": 0.08862, + "47": 0.08698, + "48": 0.089, + "49": 0.08841, + "50": 0.08813, + "51": 0.09282, + "52": 0.08991, + "53": 0.08846, + "54": 0.08878, + "55": 0.08875, + "56": 0.0897, + "57": 0.0888, + "58": 0.08814, + "59": 0.08821, + "60": 0.08782, + "61": 0.0888, + "62": 0.08762, + "63": 0.08743, + "64": 0.0879, + "65": 0.08877, + "66": 0.08656, + "67": 0.08681, + "68": 0.08654, + "69": 0.08705, + "70": 0.08667, + "71": 0.08696, + "72": 0.08664, + "73": 0.08625, + "74": 0.08667, + "75": 0.08656, + "76": 0.08557, + "77": 0.08578, + "78": 0.08586, + "79": 0.08584, + "80": 0.08576, + "81": 0.08653, + "82": 0.08572, + "83": 0.08613, + "84": 0.08557, + "85": 0.08616, + "86": 0.08608, + "87": 0.08563, + "88": 0.08581, + "89": 0.08591, + "90": 0.08593, + "91": 0.08543, + "92": 0.08641, + "93": 0.08635, + "94": 0.08549, + "95": 0.08554, + "96": 0.08487, + "97": 0.08505, + "98": 0.08522, + "99": 0.08533, + "100": 0.08544 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", "15": "nan", - "20": 1143.0, - "25": 1185.0, - "30": 1375.0, + "16": "nan", + "17": "nan", + "18": 1208.0, + "19": 1541.0, + "20": 1173.0, + "21": 1368.0, + "22": "nan", + "23": 1317.0, + "24": 1131.0, + "25": 1119.0, + "26": 1067.0, + "27": 1300.0, + "28": 1549.0, + "29": 1501.0, + "30": 1354.0, + "31": 1044.0, + "32": 1202.0, + "33": 1503.0, + "34": 1276.0, "35": 1207.0, - "40": 1423.0, - "45": 1344.0, - "50": 1559.0, - "55": 1573.0, - "60": 1564.0, - "65": 1496.0, - "70": 1898.0, - "75": 1893.0, - "80": 2134.0, - "85": 2231.0, - "90": 2378.0, - "95": 2285.0, - "100": 2233.0 + "36": 1217.0, + "37": 1430.0, + "38": 1473.0, + "39": 1525.0, + "40": 1413.0, + "41": 1441.0, + "42": 1487.0, + "43": 1151.0, + "44": 1217.0, + "45": 1314.0, + "46": 1265.0, + "47": 1868.0, + "48": 1314.0, + "49": 1105.0, + "50": 1476.0, + "51": 1508.0, + "52": 1400.0, + "53": 1740.0, + "54": 1498.0, + "55": 1551.0, + "56": 1363.0, + "57": 1465.0, + "58": 1610.0, + "59": 1574.0, + "60": 1599.0, + "61": 1727.0, + "62": 1804.0, + "63": 1590.0, + "64": 1813.0, + "65": 1398.0, + "66": 1738.0, + "67": 1536.0, + "68": 1764.0, + "69": 1781.0, + "70": 1926.0, + "71": 1950.0, + "72": 1461.0, + "73": 1985.0, + "74": 1345.0, + "75": 1871.0, + "76": 1732.0, + "77": 2086.0, + "78": 2075.0, + "79": 1992.0, + "80": 2260.0, + "81": 2300.0, + "82": 2290.0, + "83": 1774.0, + "84": 2172.0, + "85": 2216.0, + "86": 2038.0, + "87": 2741.0, + "88": 2079.0, + "89": 2349.0, + "90": 2315.0, + "91": 1875.0, + "92": 2611.0, + "93": 2053.0, + "94": 2220.0, + "95": 2296.0, + "96": 2665.0, + "97": 2516.0, + "98": 2549.0, + "99": 2378.0, + "100": 2257.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json new file mode 100644 index 00000000000..93d8ec41a7b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.83572, + "2": 10.8327, + "3": 10.83034, + "4": 10.80223, + "5": 10.85556, + "6": 10.86667, + "7": 10.84637, + "8": 10.84251, + "9": 10.85429, + "10": 10.80752, + "11": 10.88921, + "12": 10.87043, + "13": 10.87395, + "14": 10.88615, + "15": 10.79982, + "16": 10.81152, + "17": 10.78946, + "18": 10.82078, + "19": 10.80788, + "20": 10.71283, + "21": 10.68647, + "22": 10.54255, + "23": 10.71926, + "24": 10.58463, + "25": 10.54402, + "26": 10.61055, + "27": 10.61018, + "28": 10.57117, + "29": 10.59062, + "30": 10.35312, + "31": 10.0924, + "32": 10.46991, + "33": 10.45536, + "34": 10.19985, + "35": 10.26269, + "36": 10.21197, + "37": 10.33645, + "38": 10.17551, + "39": 10.39309, + "40": 10.07155, + "41": 10.13767, + "42": 10.19633, + "43": 9.81246, + "44": 9.93615, + "45": 9.81584, + "46": 9.80638, + "47": 10.12803, + "48": 9.82444, + "49": 9.50618, + "50": 9.89067 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1564.0, + "2": 1660.0, + "3": 1617.0, + "4": 1711.0, + "5": 1840.0, + "6": 1754.0, + "7": 1793.0, + "8": 1599.0, + "9": 1894.0, + "10": 1424.0, + "11": 1793.0, + "12": 1727.0, + "13": 1856.0, + "14": 1761.0, + "15": 1915.0, + "16": 1859.0, + "17": 1727.0, + "18": 1585.0, + "19": 1747.0, + "20": 1655.0, + "21": 1783.0, + "22": 1671.0, + "23": 2028.0, + "24": 1698.0, + "25": 1618.0, + "26": 1708.0, + "27": 1804.0, + "28": 1978.0, + "29": 1950.0, + "30": 1911.0, + "31": 1582.0, + "32": 1852.0, + "33": 2049.0, + "34": 1912.0, + "35": 1999.0, + "36": 1972.0, + "37": 2236.0, + "38": 2294.0, + "39": 2209.0, + "40": 2171.0, + "41": 2340.0, + "42": 2153.0, + "43": 2015.0, + "44": 2122.0, + "45": 2123.0, + "46": 2194.0, + "47": 2463.0, + "48": 2382.0, + "49": 2300.0, + "50": 2397.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 952847360.0, + "2": 952847360.0, + "3": 952847360.0, + "4": 952847360.0, + "5": 952847360.0, + "6": 952847360.0, + "7": 952847360.0, + "8": 952847360.0, + "9": 952847360.0, + "10": 952847360.0, + "11": 952847360.0, + "12": 952847360.0, + "13": 952847360.0, + "14": 952847360.0, + "15": 952847360.0, + "16": 952847360.0, + "17": 952847360.0, + "18": 952847360.0, + "19": 952847360.0, + "20": 952847360.0, + "21": 952847360.0, + "22": 952847360.0, + "23": 952847360.0, + "24": 952847360.0, + "25": 952847360.0, + "26": 952847360.0, + "27": 952847360.0, + "28": 952847360.0, + "29": 952847360.0, + "30": 952847360.0, + "31": 952847360.0, + "32": 952847360.0, + "33": 952847360.0, + "34": 952847360.0, + "35": 952847360.0, + "36": 952847360.0, + "37": 952847360.0, + "38": 952847360.0, + "39": 952847360.0, + "40": 952847360.0, + "41": 952847360.0, + "42": 952847360.0, + "43": 952847360.0, + "44": 952847360.0, + "45": 952847360.0, + "46": 952847360.0, + "47": 952847360.0, + "48": 952847360.0, + "49": 952847360.0, + "50": 952847360.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3275808768.0, + "2": 3637371904.0, + "3": 3637371904.0, + "4": 3637371904.0, + "5": 3637371904.0, + "6": 3637371904.0, + "7": 3637371904.0, + "8": 3637371904.0, + "9": 3637371904.0, + "10": 3637371904.0, + "11": 3637371904.0, + "12": 3637371904.0, + "13": 3637371904.0, + "14": 3637371904.0, + "15": 3637371904.0, + "16": 3637371904.0, + "17": 3637371904.0, + "18": 3637371904.0, + "19": 3637371904.0, + "20": 3637371904.0, + "21": 3637371904.0, + "22": 3637371904.0, + "23": 3637371904.0, + "24": 3637371904.0, + "25": 3637371904.0, + "26": 3637371904.0, + "27": 3637371904.0, + "28": 3637371904.0, + "29": 3637371904.0, + "30": 3637371904.0, + "31": 3637371904.0, + "32": 3637371904.0, + "33": 3637371904.0, + "34": 3637371904.0, + "35": 3637371904.0, + "36": 3637371904.0, + "37": 3637371904.0, + "38": 3637371904.0, + "39": 3637371904.0, + "40": 3637371904.0, + "41": 3637371904.0, + "42": 3637371904.0, + "43": 3637371904.0, + "44": 3637371904.0, + "45": 3637371904.0, + "46": 3637371904.0, + "47": 3637371904.0, + "48": 3637371904.0, + "49": 3637371904.0, + "50": 3637371904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5.51421, + "2": 0.1601, + "3": 0.13443, + "4": 0.13403, + "5": 0.13289, + "6": 0.13124, + "7": 0.13139, + "8": 0.13164, + "9": 0.12936, + "10": 0.1315, + "11": 0.13031, + "12": 0.12986, + "13": 0.13068, + "14": 0.13035, + "15": 0.12992, + "16": 0.12988, + "17": 0.12998, + "18": 0.13022, + "19": 0.13175, + "20": 0.13299, + "21": 0.13082, + "22": 0.13147, + "23": 0.13039, + "24": 0.13108, + "25": 0.13113, + "26": 0.1301, + "27": 0.12969, + "28": 0.12939, + "29": 0.13157, + "30": 0.13294, + "31": 0.132, + "32": 0.13107, + "33": 0.13092, + "34": 0.12992, + "35": 0.13029, + "36": 0.13044, + "37": 0.13082, + "38": 0.1305, + "39": 0.13098, + "40": 0.13116, + "41": 0.13121, + "42": 0.13136, + "43": 0.13117, + "44": 0.13317, + "45": 0.13163, + "46": 0.1312, + "47": 0.13036, + "48": 0.13105, + "49": 0.13091, + "50": 0.1304 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json index 45cbf767489..bdc8c7f9895 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8337, - "5": 10.85677, - "10": 10.80851, - "15": 10.79856, - "20": 10.71089, - "25": 10.5436, - "30": 10.35069, - "35": 10.26074, + "1": 10.83572, + "2": 10.8327, + "3": 10.83034, + "4": 10.80223, + "5": 10.85556, + "6": 10.86667, + "7": 10.84637, + "8": 10.84251, + "9": 10.85429, + "10": 10.80752, + "11": 10.88921, + "12": 10.87043, + "13": 10.87395, + "14": 10.88615, + "15": 10.79982, + "16": 10.81152, + "17": 10.78946, + "18": 10.82078, + "19": 10.80788, + "20": 10.71283, + "21": 10.68647, + "22": 10.54255, + "23": 10.71926, + "24": 10.58463, + "25": 10.54402, + "26": 10.61055, + "27": 10.61018, + "28": 10.57117, + "29": 10.59062, + "30": 10.35312, + "31": 10.0924, + "32": 10.46991, + "33": 10.45536, + "34": 10.19985, + "35": 10.26269, + "36": 10.21197, + "37": 10.33645, + "38": 10.17551, + "39": 10.39309, "40": 10.07155, - "45": 9.81507, - "50": 9.88952 + "41": 10.13767, + "42": 10.19633, + "43": 9.81246, + "44": 9.93615, + "45": 9.81584, + "46": 9.80638, + "47": 10.12803, + "48": 9.82444, + "49": 9.50618, + "50": 9.89067 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1563.0, - "5": 1808.0, - "10": 1377.0, - "15": 1872.0, - "20": 1675.0, - "25": 1549.0, - "30": 2043.0, - "35": 2002.0, - "40": 2311.0, - "45": 2149.0, - "50": 2368.0 + "1": 1564.0, + "2": 1660.0, + "3": 1617.0, + "4": 1711.0, + "5": 1840.0, + "6": 1754.0, + "7": 1793.0, + "8": 1599.0, + "9": 1894.0, + "10": 1424.0, + "11": 1793.0, + "12": 1727.0, + "13": 1856.0, + "14": 1761.0, + "15": 1915.0, + "16": 1859.0, + "17": 1727.0, + "18": 1585.0, + "19": 1747.0, + "20": 1655.0, + "21": 1783.0, + "22": 1671.0, + "23": 2028.0, + "24": 1698.0, + "25": 1618.0, + "26": 1708.0, + "27": 1804.0, + "28": 1978.0, + "29": 1950.0, + "30": 1911.0, + "31": 1582.0, + "32": 1852.0, + "33": 2049.0, + "34": 1912.0, + "35": 1999.0, + "36": 1972.0, + "37": 2236.0, + "38": 2294.0, + "39": 2209.0, + "40": 2171.0, + "41": 2340.0, + "42": 2153.0, + "43": 2015.0, + "44": 2122.0, + "45": 2123.0, + "46": 2194.0, + "47": 2463.0, + "48": 2382.0, + "49": 2300.0, + "50": 2397.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 948653056.0, - "5": 948653056.0, - "10": 948653056.0, - "15": 948653056.0, - "20": 948653056.0, - "25": 948653056.0, - "30": 948653056.0, - "35": 948653056.0, - "40": 948653056.0, - "45": 948653056.0, - "50": 948653056.0 + "1": 952847360.0, + "2": 952847360.0, + "3": 952847360.0, + "4": 952847360.0, + "5": 952847360.0, + "6": 952847360.0, + "7": 952847360.0, + "8": 952847360.0, + "9": 952847360.0, + "10": 952847360.0, + "11": 952847360.0, + "12": 952847360.0, + "13": 952847360.0, + "14": 952847360.0, + "15": 952847360.0, + "16": 952847360.0, + "17": 952847360.0, + "18": 952847360.0, + "19": 952847360.0, + "20": 952847360.0, + "21": 952847360.0, + "22": 952847360.0, + "23": 952847360.0, + "24": 952847360.0, + "25": 952847360.0, + "26": 952847360.0, + "27": 952847360.0, + "28": 952847360.0, + "29": 952847360.0, + "30": 952847360.0, + "31": 952847360.0, + "32": 952847360.0, + "33": 952847360.0, + "34": 952847360.0, + "35": 952847360.0, + "36": 952847360.0, + "37": 952847360.0, + "38": 952847360.0, + "39": 952847360.0, + "40": 952847360.0, + "41": 952847360.0, + "42": 952847360.0, + "43": 952847360.0, + "44": 952847360.0, + "45": 952847360.0, + "46": 952847360.0, + "47": 952847360.0, + "48": 952847360.0, + "49": 952847360.0, + "50": 952847360.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 3275284480.0, - "5": 3632653312.0, - "10": 3632653312.0, - "15": 3632653312.0, - "20": 3632653312.0, - "25": 3632653312.0, - "30": 3632653312.0, - "35": 3632653312.0, - "40": 3632653312.0, - "45": 3632653312.0, - "50": 3632653312.0 + "1": 3275808768.0, + "2": 3637371904.0, + "3": 3637371904.0, + "4": 3637371904.0, + "5": 3637371904.0, + "6": 3637371904.0, + "7": 3637371904.0, + "8": 3637371904.0, + "9": 3637371904.0, + "10": 3637371904.0, + "11": 3637371904.0, + "12": 3637371904.0, + "13": 3637371904.0, + "14": 3637371904.0, + "15": 3637371904.0, + "16": 3637371904.0, + "17": 3637371904.0, + "18": 3637371904.0, + "19": 3637371904.0, + "20": 3637371904.0, + "21": 3637371904.0, + "22": 3637371904.0, + "23": 3637371904.0, + "24": 3637371904.0, + "25": 3637371904.0, + "26": 3637371904.0, + "27": 3637371904.0, + "28": 3637371904.0, + "29": 3637371904.0, + "30": 3637371904.0, + "31": 3637371904.0, + "32": 3637371904.0, + "33": 3637371904.0, + "34": 3637371904.0, + "35": 3637371904.0, + "36": 3637371904.0, + "37": 3637371904.0, + "38": 3637371904.0, + "39": 3637371904.0, + "40": 3637371904.0, + "41": 3637371904.0, + "42": 3637371904.0, + "43": 3637371904.0, + "44": 3637371904.0, + "45": 3637371904.0, + "46": 3637371904.0, + "47": 3637371904.0, + "48": 3637371904.0, + "49": 3637371904.0, + "50": 3637371904.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.60457, - "5": 0.13759, - "10": 0.13772, - "15": 0.14601, - "20": 0.14893, - "25": 0.14839, - "30": 0.14809, - "35": 0.14777, - "40": 0.14825, - "45": 0.14788, - "50": 0.14852 + "1": 5.86972, + "2": 0.17288, + "3": 0.13781, + "4": 0.13826, + "5": 0.13569, + "6": 0.13252, + "7": 0.1323, + "8": 0.13208, + "9": 0.13237, + "10": 0.13177, + "11": 0.13164, + "12": 0.135, + "13": 0.13389, + "14": 0.13431, + "15": 0.13376, + "16": 0.1342, + "17": 0.13348, + "18": 0.13307, + "19": 0.13389, + "20": 0.13476, + "21": 0.13346, + "22": 0.13333, + "23": 0.13336, + "24": 0.13304, + "25": 0.13373, + "26": 0.13283, + "27": 0.1331, + "28": 0.13314, + "29": 0.13299, + "30": 0.13362, + "31": 0.13392, + "32": 0.13417, + "33": 0.13406, + "34": 0.13351, + "35": 0.13357, + "36": 0.13345, + "37": 0.13422, + "38": 0.1339, + "39": 0.13419, + "40": 0.13437, + "41": 0.13425, + "42": 0.13364, + "43": 0.13389, + "44": 0.13482, + "45": 0.13461, + "46": 0.134, + "47": 0.13363, + "48": 0.13416, + "49": 0.13386, + "50": 0.13343 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..f6017e22e9a --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.83568, + "2": 10.83266, + "3": 10.83151, + "4": 10.80343, + "5": 10.8567, + "6": 10.86778, + "7": 10.84836, + "8": 10.84624, + "9": 10.85924, + "10": 10.81478, + "11": 10.89821, + "12": 10.88433, + "13": 10.88963, + "14": 10.90075, + "15": 10.85098, + "16": 10.86603, + "17": 10.85455, + "18": 10.88507, + "19": 10.8773, + "20": 10.85865, + "21": 10.85654, + "22": 10.79685, + "23": 10.88724, + "24": 10.82649, + "25": 10.81343, + "26": 10.82705, + "27": 10.84612, + "28": 10.84227, + "29": 10.85329, + "30": 10.74969, + "31": 10.63041, + "32": 10.79004, + "33": 10.77234, + "34": 10.65722, + "35": 10.65857, + "36": 10.61583, + "37": 10.67536, + "38": 10.58101, + "39": 10.69083, + "40": 10.50359, + "41": 10.52777, + "42": 10.55371, + "43": 10.28636, + "44": 10.36369, + "45": 10.2738, + "46": 10.24567, + "47": 10.45103, + "48": 10.23707, + "49": 9.99555, + "50": 10.25588, + "51": 10.20129, + "52": 10.10855, + "53": 10.34609, + "54": 10.24857, + "55": 10.18782, + "56": 9.95521, + "57": 9.81221, + "58": 10.10875, + "59": 9.8863, + "60": 9.80901, + "61": 9.94824, + "62": 10.1999, + "63": 9.64431, + "64": 9.9951, + "65": 9.24475, + "66": 9.90917, + "67": 9.59735, + "68": 9.97285, + "69": 9.96332, + "70": 9.91039, + "71": 9.78596, + "72": 9.77263, + "73": 9.6618, + "74": 9.16289, + "75": 9.5812, + "76": 9.26137, + "77": 10.17615, + "78": 9.85644, + "79": 9.50644, + "80": 9.54102, + "81": 9.61313, + "82": 9.80669, + "83": 9.44696, + "84": 9.52782, + "85": 9.72633, + "86": 9.19099, + "87": 9.68736, + "88": 9.85216, + "89": 9.71335, + "90": 9.90316, + "91": 9.46064, + "92": 9.46059, + "93": 9.19418, + "94": 8.93434, + "95": 9.60258, + "96": 9.61852, + "97": 9.39594, + "98": 9.76012, + "99": 8.98668, + "100": 9.49405 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 684610560.0, + "2": 685659136.0, + "3": 685659136.0, + "4": 685659136.0, + "5": 685659136.0, + "6": 685659136.0, + "7": 685659136.0, + "8": 685659136.0, + "9": 685659136.0, + "10": 685659136.0, + "11": 685659136.0, + "12": 685659136.0, + "13": 685659136.0, + "14": 685659136.0, + "15": 685659136.0, + "16": 685659136.0, + "17": 1043027456.0, + "18": 1043027456.0, + "19": 1043027456.0, + "20": 1043027456.0, + "21": 1043027456.0, + "22": 1043027456.0, + "23": 1043027456.0, + "24": 1043027456.0, + "25": 1043027456.0, + "26": 1043027456.0, + "27": 1043027456.0, + "28": 1043027456.0, + "29": 1043027456.0, + "30": 1043027456.0, + "31": 1043027456.0, + "32": 1043027456.0, + "33": 1043027456.0, + "34": 1043027456.0, + "35": 1043027456.0, + "36": 1043027456.0, + "37": 1043027456.0, + "38": 1043027456.0, + "39": 1043027456.0, + "40": 1043027456.0, + "41": 1043027456.0, + "42": 1043027456.0, + "43": 1043027456.0, + "44": 1043027456.0, + "45": 1043027456.0, + "46": 1043027456.0, + "47": 1043027456.0, + "48": 1043027456.0, + "49": 1043027456.0, + "50": 1043027456.0, + "51": 1043027456.0, + "52": 1043027456.0, + "53": 1043027456.0, + "54": 1043027456.0, + "55": 1043027456.0, + "56": 1043027456.0, + "57": 1043027456.0, + "58": 1043027456.0, + "59": 1043027456.0, + "60": 1043027456.0, + "61": 1043027456.0, + "62": 1043027456.0, + "63": 1043027456.0, + "64": 1043027456.0, + "65": 1043027456.0, + "66": 1043027456.0, + "67": 1043027456.0, + "68": 1043027456.0, + "69": 1043027456.0, + "70": 1043027456.0, + "71": 1043027456.0, + "72": 1043027456.0, + "73": 1043027456.0, + "74": 1043027456.0, + "75": 1043027456.0, + "76": 1043027456.0, + "77": 1043027456.0, + "78": 1043027456.0, + "79": 1043027456.0, + "80": 1043027456.0, + "81": 1043027456.0, + "82": 1043027456.0, + "83": 1043027456.0, + "84": 1043027456.0, + "85": 1043027456.0, + "86": 1043027456.0, + "87": 1043027456.0, + "88": 1043027456.0, + "89": 1043027456.0, + "90": 1043027456.0, + "91": 1043027456.0, + "92": 1043027456.0, + "93": 1043027456.0, + "94": 1043027456.0, + "95": 1043027456.0, + "96": 1043027456.0, + "97": 1043027456.0, + "98": 1043027456.0, + "99": 1043027456.0, + "100": 1043027456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3187304960.0, + "2": 3187305472.0, + "3": 3187305472.0, + "4": 3187305472.0, + "5": 3187305472.0, + "6": 3187305472.0, + "7": 3187305472.0, + "8": 3187305472.0, + "9": 3187305472.0, + "10": 3187305472.0, + "11": 3187305472.0, + "12": 3187305472.0, + "13": 3187305472.0, + "14": 3187305472.0, + "15": 3187305472.0, + "16": 3187305472.0, + "17": 3187305472.0, + "18": 3544935936.0, + "19": 3544935936.0, + "20": 3544935936.0, + "21": 3544935936.0, + "22": 3544935936.0, + "23": 3544935936.0, + "24": 3544935936.0, + "25": 3544935936.0, + "26": 3544935936.0, + "27": 3544935936.0, + "28": 3544935936.0, + "29": 3544935936.0, + "30": 3544935936.0, + "31": 3544935936.0, + "32": 3544935936.0, + "33": 3544935936.0, + "34": 3544935936.0, + "35": 3544935936.0, + "36": 3544935936.0, + "37": 3544935936.0, + "38": 3544935936.0, + "39": 3544935936.0, + "40": 3544935936.0, + "41": 3544935936.0, + "42": 3544935936.0, + "43": 3544935936.0, + "44": 3544935936.0, + "45": 3544935936.0, + "46": 3544935936.0, + "47": 3544935936.0, + "48": 3544935936.0, + "49": 3544935936.0, + "50": 3544935936.0, + "51": 3544935936.0, + "52": 3544935936.0, + "53": 3544935936.0, + "54": 3544935936.0, + "55": 3544935936.0, + "56": 3544935936.0, + "57": 3544935936.0, + "58": 3544935936.0, + "59": 3544935936.0, + "60": 3544935936.0, + "61": 3544935936.0, + "62": 3544935936.0, + "63": 3544935936.0, + "64": 3544935936.0, + "65": 3544935936.0, + "66": 3544935936.0, + "67": 3544935936.0, + "68": 3544935936.0, + "69": 3544935936.0, + "70": 3544935936.0, + "71": 3544935936.0, + "72": 3544935936.0, + "73": 3544935936.0, + "74": 3544935936.0, + "75": 3544935936.0, + "76": 3544935936.0, + "77": 3544935936.0, + "78": 3544935936.0, + "79": 3544935936.0, + "80": 3544935936.0, + "81": 3544935936.0, + "82": 3544935936.0, + "83": 3544935936.0, + "84": 3544935936.0, + "85": 3544935936.0, + "86": 3544935936.0, + "87": 3544935936.0, + "88": 3544935936.0, + "89": 3544935936.0, + "90": 3544935936.0, + "91": 3544935936.0, + "92": 3544935936.0, + "93": 3544935936.0, + "94": 3544935936.0, + "95": 3544935936.0, + "96": 3544935936.0, + "97": 3544935936.0, + "98": 3544935936.0, + "99": 3544935936.0, + "100": 3544935936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.71216, + "2": 0.14973, + "3": 0.11823, + "4": 0.11589, + "5": 0.115, + "6": 0.11544, + "7": 0.11494, + "8": 0.11629, + "9": 0.11558, + "10": 0.11559, + "11": 0.11587, + "12": 0.11557, + "13": 0.11565, + "14": 0.11485, + "15": 0.11461, + "16": 0.11592, + "17": 0.17863, + "18": 0.13323, + "19": 0.12474, + "20": 0.12384, + "21": 0.12332, + "22": 0.11528, + "23": 0.12299, + "24": 0.12347, + "25": 0.12391, + "26": 0.12403, + "27": 0.12332, + "28": 0.12373, + "29": 0.12271, + "30": 0.12236, + "31": 0.12244, + "32": 0.12402, + "33": 0.12212, + "34": 0.12202, + "35": 0.1228, + "36": 0.12235, + "37": 0.12236, + "38": 0.12224, + "39": 0.12198, + "40": 0.12302, + "41": 0.12287, + "42": 0.1224, + "43": 0.12222, + "44": 0.12187, + "45": 0.12269, + "46": 0.1224, + "47": 0.12251, + "48": 0.12187, + "49": 0.12239, + "50": 0.12176, + "51": 0.12901, + "52": 0.12249, + "53": 0.12142, + "54": 0.12162, + "55": 0.12146, + "56": 0.12102, + "57": 0.12365, + "58": 0.12178, + "59": 0.12229, + "60": 0.12129, + "61": 0.12124, + "62": 0.12162, + "63": 0.12218, + "64": 0.12145, + "65": 0.1222, + "66": 0.12127, + "67": 0.12238, + "68": 0.12143, + "69": 0.12118, + "70": 0.12095, + "71": 0.12155, + "72": 0.12227, + "73": 0.12265, + "74": 0.1222, + "75": 0.1232, + "76": 0.12175, + "77": 0.12226, + "78": 0.12166, + "79": 0.12163, + "80": 0.12235, + "81": 0.12164, + "82": 0.1221, + "83": 0.12172, + "84": 0.12179, + "85": 0.1221, + "86": 0.12134, + "87": 0.12203, + "88": 0.12166, + "89": 0.12224, + "90": 0.1218, + "91": 0.12135, + "92": 0.12216, + "93": 0.12267, + "94": 0.12146, + "95": 0.1222, + "96": 0.12287, + "97": 0.12296, + "98": 0.12234, + "99": 0.12184, + "100": 0.12221 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2249.0, + "18": 2165.0, + "19": 2362.0, + "20": 1953.0, + "21": 1898.0, + "22": "nan", + "23": 2371.0, + "24": 1984.0, + "25": 1818.0, + "26": 1980.0, + "27": 2078.0, + "28": 2467.0, + "29": 2395.0, + "30": 2298.0, + "31": 1682.0, + "32": 2236.0, + "33": 2192.0, + "34": 1800.0, + "35": 2083.0, + "36": 2139.0, + "37": 2498.0, + "38": 2218.0, + "39": 2642.0, + "40": 2287.0, + "41": 2344.0, + "42": 2304.0, + "43": 2098.0, + "44": 2107.0, + "45": 2243.0, + "46": 1960.0, + "47": 2729.0, + "48": 2418.0, + "49": 1910.0, + "50": 2426.0, + "51": 2335.0, + "52": 2407.0, + "53": 2888.0, + "54": 2477.0, + "55": 2440.0, + "56": 2286.0, + "57": 2340.0, + "58": 2652.0, + "59": 2321.0, + "60": 2493.0, + "61": 2812.0, + "62": 2711.0, + "63": 2367.0, + "64": 2802.0, + "65": 2411.0, + "66": 2869.0, + "67": 2577.0, + "68": 2859.0, + "69": 2524.0, + "70": 3119.0, + "71": 2926.0, + "72": 2251.0, + "73": 2929.0, + "74": 2110.0, + "75": 2884.0, + "76": 2992.0, + "77": 3380.0, + "78": 3484.0, + "79": 3533.0, + "80": 3549.0, + "81": 3616.0, + "82": 3347.0, + "83": 3124.0, + "84": 3276.0, + "85": 3721.0, + "86": 3207.0, + "87": 3941.0, + "88": 3250.0, + "89": 3863.0, + "90": 3452.0, + "91": 2630.0, + "92": 3431.0, + "93": 3123.0, + "94": 3671.0, + "95": 3340.0, + "96": 3874.0, + "97": 3519.0, + "98": 3727.0, + "99": 3447.0, + "100": 3338.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json index 56c78e6ebf6..094be8516a7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8337, - "5": 10.85789, - "10": 10.81578, - "15": 10.84971, - "20": 10.85686, - "25": 10.81326, - "30": 10.74663, - "35": 10.65647, - "40": 10.50351, - "45": 10.27258, - "50": 10.25445, - "55": 10.18699, - "60": 9.8098, - "65": 9.24409, - "70": 9.91024, - "75": 9.58173, - "80": 9.54167, - "85": 9.7262, - "90": 9.90242, - "95": 9.60208, - "100": 9.49424 + "1": 10.83568, + "2": 10.83266, + "3": 10.83151, + "4": 10.80343, + "5": 10.8567, + "6": 10.86778, + "7": 10.84836, + "8": 10.84624, + "9": 10.85924, + "10": 10.81478, + "11": 10.89821, + "12": 10.88433, + "13": 10.88963, + "14": 10.90075, + "15": 10.85098, + "16": 10.86603, + "17": 10.85455, + "18": 10.88507, + "19": 10.8773, + "20": 10.85865, + "21": 10.85654, + "22": 10.79685, + "23": 10.88724, + "24": 10.82649, + "25": 10.81343, + "26": 10.82705, + "27": 10.84612, + "28": 10.84227, + "29": 10.85329, + "30": 10.74969, + "31": 10.63041, + "32": 10.79004, + "33": 10.77234, + "34": 10.65722, + "35": 10.65857, + "36": 10.61583, + "37": 10.67536, + "38": 10.58101, + "39": 10.69083, + "40": 10.50359, + "41": 10.52777, + "42": 10.55371, + "43": 10.28636, + "44": 10.36369, + "45": 10.2738, + "46": 10.24567, + "47": 10.45103, + "48": 10.23707, + "49": 9.99555, + "50": 10.25588, + "51": 10.20129, + "52": 10.10855, + "53": 10.34609, + "54": 10.24857, + "55": 10.18782, + "56": 9.95521, + "57": 9.81221, + "58": 10.10875, + "59": 9.8863, + "60": 9.80901, + "61": 9.94824, + "62": 10.1999, + "63": 9.64431, + "64": 9.9951, + "65": 9.24475, + "66": 9.90917, + "67": 9.59735, + "68": 9.97285, + "69": 9.96332, + "70": 9.91039, + "71": 9.78596, + "72": 9.77263, + "73": 9.6618, + "74": 9.16289, + "75": 9.5812, + "76": 9.26137, + "77": 10.17615, + "78": 9.85644, + "79": 9.50644, + "80": 9.54102, + "81": 9.61313, + "82": 9.80669, + "83": 9.44696, + "84": 9.52782, + "85": 9.72633, + "86": 9.19099, + "87": 9.68736, + "88": 9.85216, + "89": 9.71335, + "90": 9.90316, + "91": 9.46064, + "92": 9.46059, + "93": 9.19418, + "94": 8.93434, + "95": 9.60258, + "96": 9.61852, + "97": 9.39594, + "98": 9.76012, + "99": 8.98668, + "100": 9.49405 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 684610560.0, - "5": 684610560.0, - "10": 684610560.0, - "15": 684610560.0, + "2": 685659136.0, + "3": 685659136.0, + "4": 685659136.0, + "5": 685659136.0, + "6": 685659136.0, + "7": 685659136.0, + "8": 685659136.0, + "9": 685659136.0, + "10": 685659136.0, + "11": 685659136.0, + "12": 685659136.0, + "13": 685659136.0, + "14": 685659136.0, + "15": 685659136.0, + "16": 685659136.0, + "17": 1043027456.0, + "18": 1043027456.0, + "19": 1043027456.0, "20": 1043027456.0, + "21": 1043027456.0, + "22": 1043027456.0, + "23": 1043027456.0, + "24": 1043027456.0, "25": 1043027456.0, + "26": 1043027456.0, + "27": 1043027456.0, + "28": 1043027456.0, + "29": 1043027456.0, "30": 1043027456.0, + "31": 1043027456.0, + "32": 1043027456.0, + "33": 1043027456.0, + "34": 1043027456.0, "35": 1043027456.0, + "36": 1043027456.0, + "37": 1043027456.0, + "38": 1043027456.0, + "39": 1043027456.0, "40": 1043027456.0, + "41": 1043027456.0, + "42": 1043027456.0, + "43": 1043027456.0, + "44": 1043027456.0, "45": 1043027456.0, + "46": 1043027456.0, + "47": 1043027456.0, + "48": 1043027456.0, + "49": 1043027456.0, "50": 1043027456.0, + "51": 1043027456.0, + "52": 1043027456.0, + "53": 1043027456.0, + "54": 1043027456.0, "55": 1043027456.0, + "56": 1043027456.0, + "57": 1043027456.0, + "58": 1043027456.0, + "59": 1043027456.0, "60": 1043027456.0, + "61": 1043027456.0, + "62": 1043027456.0, + "63": 1043027456.0, + "64": 1043027456.0, "65": 1043027456.0, + "66": 1043027456.0, + "67": 1043027456.0, + "68": 1043027456.0, + "69": 1043027456.0, "70": 1043027456.0, + "71": 1043027456.0, + "72": 1043027456.0, + "73": 1043027456.0, + "74": 1043027456.0, "75": 1043027456.0, + "76": 1043027456.0, + "77": 1043027456.0, + "78": 1043027456.0, + "79": 1043027456.0, "80": 1043027456.0, + "81": 1043027456.0, + "82": 1043027456.0, + "83": 1043027456.0, + "84": 1043027456.0, "85": 1043027456.0, + "86": 1043027456.0, + "87": 1043027456.0, + "88": 1043027456.0, + "89": 1043027456.0, "90": 1043027456.0, + "91": 1043027456.0, + "92": 1043027456.0, + "93": 1043027456.0, + "94": 1043027456.0, "95": 1043027456.0, + "96": 1043027456.0, + "97": 1043027456.0, + "98": 1043027456.0, + "99": 1043027456.0, "100": 1043027456.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 3187304960.0, + "2": 3187305472.0, + "3": 3187305472.0, + "4": 3187305472.0, "5": 3187305472.0, + "6": 3187305472.0, + "7": 3187305472.0, + "8": 3187305472.0, + "9": 3187305472.0, "10": 3187305472.0, + "11": 3187305472.0, + "12": 3187305472.0, + "13": 3187305472.0, + "14": 3187305472.0, "15": 3187305472.0, - "20": 3547033088.0, - "25": 3547033088.0, - "30": 3547033088.0, - "35": 3547033088.0, - "40": 3547033088.0, - "45": 3547033088.0, - "50": 3547033088.0, - "55": 3547033088.0, - "60": 3547033088.0, - "65": 3547033088.0, - "70": 3547033088.0, - "75": 3547033088.0, - "80": 3547033088.0, - "85": 3547033088.0, - "90": 3547033088.0, - "95": 3547033088.0, - "100": 3547033088.0 + "16": 3187305472.0, + "17": 3187305472.0, + "18": 3544935936.0, + "19": 3544935936.0, + "20": 3544935936.0, + "21": 3544935936.0, + "22": 3544935936.0, + "23": 3544935936.0, + "24": 3544935936.0, + "25": 3544935936.0, + "26": 3544935936.0, + "27": 3544935936.0, + "28": 3544935936.0, + "29": 3544935936.0, + "30": 3544935936.0, + "31": 3544935936.0, + "32": 3544935936.0, + "33": 3544935936.0, + "34": 3544935936.0, + "35": 3544935936.0, + "36": 3544935936.0, + "37": 3544935936.0, + "38": 3544935936.0, + "39": 3544935936.0, + "40": 3544935936.0, + "41": 3544935936.0, + "42": 3544935936.0, + "43": 3544935936.0, + "44": 3544935936.0, + "45": 3544935936.0, + "46": 3544935936.0, + "47": 3544935936.0, + "48": 3544935936.0, + "49": 3544935936.0, + "50": 3544935936.0, + "51": 3544935936.0, + "52": 3544935936.0, + "53": 3544935936.0, + "54": 3544935936.0, + "55": 3544935936.0, + "56": 3544935936.0, + "57": 3544935936.0, + "58": 3544935936.0, + "59": 3544935936.0, + "60": 3544935936.0, + "61": 3544935936.0, + "62": 3544935936.0, + "63": 3544935936.0, + "64": 3544935936.0, + "65": 3544935936.0, + "66": 3544935936.0, + "67": 3544935936.0, + "68": 3544935936.0, + "69": 3544935936.0, + "70": 3544935936.0, + "71": 3544935936.0, + "72": 3544935936.0, + "73": 3544935936.0, + "74": 3544935936.0, + "75": 3544935936.0, + "76": 3544935936.0, + "77": 3544935936.0, + "78": 3544935936.0, + "79": 3544935936.0, + "80": 3544935936.0, + "81": 3544935936.0, + "82": 3544935936.0, + "83": 3544935936.0, + "84": 3544935936.0, + "85": 3544935936.0, + "86": 3544935936.0, + "87": 3544935936.0, + "88": 3544935936.0, + "89": 3544935936.0, + "90": 3544935936.0, + "91": 3544935936.0, + "92": 3544935936.0, + "93": 3544935936.0, + "94": 3544935936.0, + "95": 3544935936.0, + "96": 3544935936.0, + "97": 3544935936.0, + "98": 3544935936.0, + "99": 3544935936.0, + "100": 3544935936.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.71226, - "5": 0.14227, - "10": 0.14747, - "15": 0.14757, - "20": 0.1588, - "25": 0.15692, - "30": 0.15748, - "35": 0.15684, - "40": 0.15704, - "45": 0.15798, - "50": 0.15606, - "55": 0.15074, - "60": 0.15245, - "65": 0.15193, - "70": 0.15162, - "75": 0.14856, - "80": 0.14952, - "85": 0.14876, - "90": 0.14933, - "95": 0.15012, - "100": 0.15591 + "1": 7.5468, + "2": 0.1514, + "3": 0.11679, + "4": 0.11442, + "5": 0.11418, + "6": 0.1134, + "7": 0.11341, + "8": 0.11355, + "9": 0.11332, + "10": 0.11336, + "11": 0.11414, + "12": 0.11322, + "13": 0.11309, + "14": 0.11355, + "15": 0.11296, + "16": 0.11311, + "17": 0.19183, + "18": 0.13278, + "19": 0.12368, + "20": 0.1244, + "21": 0.12354, + "22": 0.11533, + "23": 0.12281, + "24": 0.12403, + "25": 0.12406, + "26": 0.12339, + "27": 0.12448, + "28": 0.12265, + "29": 0.1229, + "30": 0.1231, + "31": 0.12325, + "32": 0.12261, + "33": 0.12283, + "34": 0.12275, + "35": 0.12311, + "36": 0.12273, + "37": 0.12367, + "38": 0.12288, + "39": 0.12297, + "40": 0.12264, + "41": 0.1206, + "42": 0.12099, + "43": 0.12152, + "44": 0.12016, + "45": 0.12042, + "46": 0.12101, + "47": 0.12019, + "48": 0.12057, + "49": 0.12054, + "50": 0.12043, + "51": 0.12804, + "52": 0.12188, + "53": 0.12082, + "54": 0.12046, + "55": 0.12243, + "56": 0.12099, + "57": 0.12158, + "58": 0.12118, + "59": 0.12094, + "60": 0.12085, + "61": 0.12158, + "62": 0.12129, + "63": 0.12239, + "64": 0.12127, + "65": 0.12091, + "66": 0.12161, + "67": 0.12115, + "68": 0.12107, + "69": 0.12194, + "70": 0.12208, + "71": 0.12158, + "72": 0.12253, + "73": 0.12311, + "74": 0.12157, + "75": 0.12129, + "76": 0.12243, + "77": 0.1209, + "78": 0.12118, + "79": 0.12236, + "80": 0.12456, + "81": 0.12169, + "82": 0.12201, + "83": 0.12239, + "84": 0.12311, + "85": 0.12253, + "86": 0.12237, + "87": 0.12156, + "88": 0.12306, + "89": 0.12961, + "90": 0.12349, + "91": 0.12189, + "92": 0.12121, + "93": 0.12178, + "94": 0.12615, + "95": 0.12189, + "96": 0.12145, + "97": 0.12112, + "98": 0.12242, + "99": 0.12142, + "100": 0.12094 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", "15": "nan", - "20": 1969.0, - "25": 1863.0, - "30": 2312.0, - "35": 2063.0, - "40": 2212.0, - "45": 2333.0, - "50": 2369.0, - "55": 2357.0, - "60": 2500.0, - "65": 2438.0, - "70": 3075.0, - "75": 2803.0, - "80": 3568.0, - "85": 3675.0, - "90": 3397.0, - "95": 3468.0, - "100": 3267.0 + "16": "nan", + "17": 2249.0, + "18": 2165.0, + "19": 2362.0, + "20": 1953.0, + "21": 1898.0, + "22": "nan", + "23": 2371.0, + "24": 1984.0, + "25": 1818.0, + "26": 1980.0, + "27": 2078.0, + "28": 2467.0, + "29": 2395.0, + "30": 2298.0, + "31": 1682.0, + "32": 2236.0, + "33": 2192.0, + "34": 1800.0, + "35": 2083.0, + "36": 2139.0, + "37": 2498.0, + "38": 2218.0, + "39": 2642.0, + "40": 2287.0, + "41": 2344.0, + "42": 2304.0, + "43": 2098.0, + "44": 2107.0, + "45": 2243.0, + "46": 1960.0, + "47": 2729.0, + "48": 2418.0, + "49": 1910.0, + "50": 2426.0, + "51": 2335.0, + "52": 2407.0, + "53": 2888.0, + "54": 2477.0, + "55": 2440.0, + "56": 2286.0, + "57": 2340.0, + "58": 2652.0, + "59": 2321.0, + "60": 2493.0, + "61": 2812.0, + "62": 2711.0, + "63": 2367.0, + "64": 2802.0, + "65": 2411.0, + "66": 2869.0, + "67": 2577.0, + "68": 2859.0, + "69": 2524.0, + "70": 3119.0, + "71": 2926.0, + "72": 2251.0, + "73": 2929.0, + "74": 2110.0, + "75": 2884.0, + "76": 2992.0, + "77": 3380.0, + "78": 3484.0, + "79": 3533.0, + "80": 3549.0, + "81": 3616.0, + "82": 3347.0, + "83": 3124.0, + "84": 3276.0, + "85": 3721.0, + "86": 3207.0, + "87": 3941.0, + "88": 3250.0, + "89": 3863.0, + "90": 3452.0, + "91": 2630.0, + "92": 3431.0, + "93": 3123.0, + "94": 3671.0, + "95": 3340.0, + "96": 3874.0, + "97": 3519.0, + "98": 3727.0, + "99": 3447.0, + "100": 3338.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json new file mode 100644 index 00000000000..ed50a469580 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79432, + "2": 10.80634, + "3": 10.80586, + "4": 10.77237, + "5": 10.84693, + "6": 10.86605, + "7": 10.82692, + "8": 10.81129, + "9": 10.8342, + "10": 10.76876, + "11": 10.89167, + "12": 10.84477, + "13": 10.85758, + "14": 10.88197, + "15": 10.78831, + "16": 10.7793, + "17": 10.75446, + "18": 10.79296, + "19": 10.79494, + "20": 10.67877, + "21": 10.65858, + "22": 10.50081, + "23": 10.71065, + "24": 10.55089, + "25": 10.50321, + "26": 10.58033, + "27": 10.58262, + "28": 10.55556, + "29": 10.56003, + "30": 10.32992, + "31": 10.08344, + "32": 10.44434, + "33": 10.44238, + "34": 10.19765, + "35": 10.25197, + "36": 10.19117, + "37": 10.31772, + "38": 10.1631, + "39": 10.37486, + "40": 10.05284, + "41": 10.1344, + "42": 10.18877, + "43": 9.80641, + "44": 9.92687, + "45": 9.80332, + "46": 9.811, + "47": 10.12605, + "48": 9.82455, + "49": 9.50975, + "50": 9.88831 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1620.0, + "2": 1664.0, + "3": 1748.0, + "4": 1757.0, + "5": 1754.0, + "6": 1841.0, + "7": 1877.0, + "8": 1697.0, + "9": 1886.0, + "10": 1404.0, + "11": 1889.0, + "12": 1689.0, + "13": 1918.0, + "14": 1757.0, + "15": 1913.0, + "16": 1860.0, + "17": 1797.0, + "18": 1655.0, + "19": 1784.0, + "20": 1616.0, + "21": 1887.0, + "22": 1751.0, + "23": 2100.0, + "24": 1717.0, + "25": 1696.0, + "26": 1723.0, + "27": 1819.0, + "28": 1980.0, + "29": 1962.0, + "30": 2046.0, + "31": 1562.0, + "32": 1935.0, + "33": 2182.0, + "34": 1919.0, + "35": 1994.0, + "36": 1947.0, + "37": 2436.0, + "38": 2218.0, + "39": 2319.0, + "40": 2278.0, + "41": 2348.0, + "42": 2258.0, + "43": 1967.0, + "44": 2011.0, + "45": 2215.0, + "46": 2291.0, + "47": 2519.0, + "48": 2517.0, + "49": 2334.0, + "50": 2325.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 779997184.0, + "2": 779997184.0, + "3": 779997184.0, + "4": 779997184.0, + "5": 779997184.0, + "6": 779997184.0, + "7": 779997184.0, + "8": 779997184.0, + "9": 779997184.0, + "10": 779997184.0, + "11": 779997184.0, + "12": 779997184.0, + "13": 779997184.0, + "14": 779997184.0, + "15": 779997184.0, + "16": 779997184.0, + "17": 779997184.0, + "18": 779997184.0, + "19": 779997184.0, + "20": 779997184.0, + "21": 779997184.0, + "22": 779997184.0, + "23": 779997184.0, + "24": 779997184.0, + "25": 779997184.0, + "26": 779997184.0, + "27": 779997184.0, + "28": 779997184.0, + "29": 779997184.0, + "30": 779997184.0, + "31": 779997184.0, + "32": 779997184.0, + "33": 779997184.0, + "34": 779997184.0, + "35": 779997184.0, + "36": 779997184.0, + "37": 779997184.0, + "38": 779997184.0, + "39": 779997184.0, + "40": 779997184.0, + "41": 779997184.0, + "42": 779997184.0, + "43": 779997184.0, + "44": 779997184.0, + "45": 779997184.0, + "46": 779997184.0, + "47": 779997184.0, + "48": 779997184.0, + "49": 779997184.0, + "50": 779997184.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2463815168.0, + "2": 2746575872.0, + "3": 2746575872.0, + "4": 2746575872.0, + "5": 2746575872.0, + "6": 2746575872.0, + "7": 2746575872.0, + "8": 2746575872.0, + "9": 2746575872.0, + "10": 2746575872.0, + "11": 2746575872.0, + "12": 2746575872.0, + "13": 2746575872.0, + "14": 2746575872.0, + "15": 2746575872.0, + "16": 2746575872.0, + "17": 2746575872.0, + "18": 2746575872.0, + "19": 2746575872.0, + "20": 2746575872.0, + "21": 2746575872.0, + "22": 2746575872.0, + "23": 2746575872.0, + "24": 2746575872.0, + "25": 2746575872.0, + "26": 2746575872.0, + "27": 2746575872.0, + "28": 2746575872.0, + "29": 2746575872.0, + "30": 2746575872.0, + "31": 2746575872.0, + "32": 2746575872.0, + "33": 2746575872.0, + "34": 2746575872.0, + "35": 2746575872.0, + "36": 2746575872.0, + "37": 2746575872.0, + "38": 2746575872.0, + "39": 2746575872.0, + "40": 2746575872.0, + "41": 2746575872.0, + "42": 2746575872.0, + "43": 2746575872.0, + "44": 2746575872.0, + "45": 2746575872.0, + "46": 2746575872.0, + "47": 2746575872.0, + "48": 2746575872.0, + "49": 2746575872.0, + "50": 2746575872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.01512, + "2": 0.18918, + "3": 0.2837, + "4": 0.16138, + "5": 0.1616, + "6": 0.16078, + "7": 0.16042, + "8": 0.15997, + "9": 0.16098, + "10": 0.16102, + "11": 0.16098, + "12": 0.16041, + "13": 0.16091, + "14": 0.16108, + "15": 0.16115, + "16": 0.16297, + "17": 0.16105, + "18": 0.16124, + "19": 0.16111, + "20": 0.16078, + "21": 0.16059, + "22": 0.16143, + "23": 0.1618, + "24": 0.16075, + "25": 0.1607, + "26": 0.16058, + "27": 0.16134, + "28": 0.16032, + "29": 0.16066, + "30": 0.1619, + "31": 0.16069, + "32": 0.16055, + "33": 0.16041, + "34": 0.16109, + "35": 0.1616, + "36": 0.16, + "37": 0.16147, + "38": 0.16094, + "39": 0.16098, + "40": 0.16007, + "41": 0.16179, + "42": 0.1607, + "43": 0.16132, + "44": 0.16157, + "45": 0.16066, + "46": 0.16082, + "47": 0.16056, + "48": 0.16095, + "49": 0.16092, + "50": 0.16103 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json index e7e40a40328..06342d2a540 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.79203, - "5": 10.84593, - "10": 10.76951, - "15": 10.78971, - "20": 10.67886, - "25": 10.50431, - "30": 10.33089, - "35": 10.25278, - "40": 10.05241, - "45": 9.80259, - "50": 9.88804 + "1": 10.79432, + "2": 10.80634, + "3": 10.80586, + "4": 10.77237, + "5": 10.84693, + "6": 10.86605, + "7": 10.82692, + "8": 10.81129, + "9": 10.8342, + "10": 10.76876, + "11": 10.89167, + "12": 10.84477, + "13": 10.85758, + "14": 10.88197, + "15": 10.78831, + "16": 10.7793, + "17": 10.75446, + "18": 10.79296, + "19": 10.79494, + "20": 10.67877, + "21": 10.65858, + "22": 10.50081, + "23": 10.71065, + "24": 10.55089, + "25": 10.50321, + "26": 10.58033, + "27": 10.58262, + "28": 10.55556, + "29": 10.56003, + "30": 10.32992, + "31": 10.08344, + "32": 10.44434, + "33": 10.44238, + "34": 10.19765, + "35": 10.25197, + "36": 10.19117, + "37": 10.31772, + "38": 10.1631, + "39": 10.37486, + "40": 10.05284, + "41": 10.1344, + "42": 10.18877, + "43": 9.80641, + "44": 9.92687, + "45": 9.80332, + "46": 9.811, + "47": 10.12605, + "48": 9.82455, + "49": 9.50975, + "50": 9.88831 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1649.0, - "5": 1890.0, - "10": 1397.0, - "15": 1928.0, - "20": 1565.0, - "25": 1699.0, - "30": 1943.0, - "35": 2018.0, - "40": 2191.0, - "45": 2207.0, - "50": 2362.0 + "1": 1620.0, + "2": 1664.0, + "3": 1748.0, + "4": 1757.0, + "5": 1754.0, + "6": 1841.0, + "7": 1877.0, + "8": 1697.0, + "9": 1886.0, + "10": 1404.0, + "11": 1889.0, + "12": 1689.0, + "13": 1918.0, + "14": 1757.0, + "15": 1913.0, + "16": 1860.0, + "17": 1797.0, + "18": 1655.0, + "19": 1784.0, + "20": 1616.0, + "21": 1887.0, + "22": 1751.0, + "23": 2100.0, + "24": 1717.0, + "25": 1696.0, + "26": 1723.0, + "27": 1819.0, + "28": 1980.0, + "29": 1962.0, + "30": 2046.0, + "31": 1562.0, + "32": 1935.0, + "33": 2182.0, + "34": 1919.0, + "35": 1994.0, + "36": 1947.0, + "37": 2436.0, + "38": 2218.0, + "39": 2319.0, + "40": 2278.0, + "41": 2348.0, + "42": 2258.0, + "43": 1967.0, + "44": 2011.0, + "45": 2215.0, + "46": 2291.0, + "47": 2519.0, + "48": 2517.0, + "49": 2334.0, + "50": 2325.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 782094336.0, - "5": 782094336.0, - "10": 782094336.0, - "15": 782094336.0, - "20": 782094336.0, - "25": 782094336.0, - "30": 782094336.0, - "35": 782094336.0, - "40": 782094336.0, - "45": 782094336.0, - "50": 782094336.0 + "1": 777900032.0, + "2": 777900032.0, + "3": 777900032.0, + "4": 777900032.0, + "5": 777900032.0, + "6": 777900032.0, + "7": 777900032.0, + "8": 777900032.0, + "9": 777900032.0, + "10": 777900032.0, + "11": 777900032.0, + "12": 777900032.0, + "13": 777900032.0, + "14": 777900032.0, + "15": 777900032.0, + "16": 777900032.0, + "17": 777900032.0, + "18": 777900032.0, + "19": 777900032.0, + "20": 777900032.0, + "21": 777900032.0, + "22": 777900032.0, + "23": 777900032.0, + "24": 777900032.0, + "25": 777900032.0, + "26": 777900032.0, + "27": 777900032.0, + "28": 777900032.0, + "29": 777900032.0, + "30": 777900032.0, + "31": 777900032.0, + "32": 777900032.0, + "33": 777900032.0, + "34": 777900032.0, + "35": 777900032.0, + "36": 777900032.0, + "37": 777900032.0, + "38": 777900032.0, + "39": 777900032.0, + "40": 777900032.0, + "41": 777900032.0, + "42": 777900032.0, + "43": 777900032.0, + "44": 777900032.0, + "45": 777900032.0, + "46": 777900032.0, + "47": 777900032.0, + "48": 777900032.0, + "49": 777900032.0, + "50": 777900032.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 2462767104.0, - "5": 2748673024.0, - "10": 2748673024.0, - "15": 2748673024.0, - "20": 2748673024.0, - "25": 2748673024.0, - "30": 2748673024.0, - "35": 2748673024.0, - "40": 2748673024.0, - "45": 2748673024.0, - "50": 2748673024.0 + "2": 2744478720.0, + "3": 2744478720.0, + "4": 2744478720.0, + "5": 2744478720.0, + "6": 2744478720.0, + "7": 2744478720.0, + "8": 2744478720.0, + "9": 2744478720.0, + "10": 2744478720.0, + "11": 2744478720.0, + "12": 2744478720.0, + "13": 2744478720.0, + "14": 2744478720.0, + "15": 2744478720.0, + "16": 2744478720.0, + "17": 2744478720.0, + "18": 2744478720.0, + "19": 2744478720.0, + "20": 2744478720.0, + "21": 2744478720.0, + "22": 2744478720.0, + "23": 2744478720.0, + "24": 2744478720.0, + "25": 2744478720.0, + "26": 2744478720.0, + "27": 2744478720.0, + "28": 2744478720.0, + "29": 2744478720.0, + "30": 2744478720.0, + "31": 2744478720.0, + "32": 2744478720.0, + "33": 2744478720.0, + "34": 2744478720.0, + "35": 2744478720.0, + "36": 2744478720.0, + "37": 2744478720.0, + "38": 2744478720.0, + "39": 2744478720.0, + "40": 2744478720.0, + "41": 2744478720.0, + "42": 2744478720.0, + "43": 2744478720.0, + "44": 2744478720.0, + "45": 2744478720.0, + "46": 2744478720.0, + "47": 2744478720.0, + "48": 2744478720.0, + "49": 2744478720.0, + "50": 2744478720.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 16.61466, - "5": 0.16864, - "10": 0.16989, - "15": 0.1682, - "20": 0.16954, - "25": 0.16878, - "30": 0.2008, - "35": 0.18359, - "40": 0.16981, - "45": 0.17863, - "50": 0.18407 + "1": 9.69348, + "2": 0.20058, + "3": 0.16793, + "4": 0.16851, + "5": 0.16769, + "6": 0.16776, + "7": 0.1679, + "8": 0.1698, + "9": 0.16773, + "10": 0.16689, + "11": 0.16616, + "12": 0.16649, + "13": 0.16602, + "14": 0.16651, + "15": 0.16681, + "16": 0.16794, + "17": 0.17068, + "18": 0.16616, + "19": 0.16604, + "20": 0.16664, + "21": 0.16675, + "22": 0.16587, + "23": 0.16669, + "24": 0.16593, + "25": 0.16666, + "26": 0.16624, + "27": 0.16546, + "28": 0.16503, + "29": 0.16469, + "30": 0.1651, + "31": 0.16508, + "32": 0.16533, + "33": 0.16475, + "34": 0.16518, + "35": 0.16543, + "36": 0.16422, + "37": 0.1648, + "38": 0.16453, + "39": 0.16423, + "40": 0.16482, + "41": 0.16457, + "42": 0.1653, + "43": 0.16536, + "44": 0.16541, + "45": 0.16481, + "46": 0.16481, + "47": 0.16542, + "48": 0.16607, + "49": 0.1639, + "50": 0.1641 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..44dd2d42336 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79449, + "2": 10.80656, + "3": 10.80727, + "4": 10.77389, + "5": 10.84829, + "6": 10.86736, + "7": 10.82922, + "8": 10.81537, + "9": 10.83956, + "10": 10.77652, + "11": 10.90107, + "12": 10.85927, + "13": 10.87396, + "14": 10.89723, + "15": 10.83961, + "16": 10.83508, + "17": 10.82101, + "18": 10.86029, + "19": 10.86558, + "20": 10.82896, + "21": 10.83275, + "22": 10.75286, + "23": 10.88062, + "24": 10.78219, + "25": 10.76607, + "26": 10.79522, + "27": 10.79866, + "28": 10.81697, + "29": 10.82169, + "30": 10.69891, + "31": 10.55698, + "32": 10.75759, + "33": 10.74362, + "34": 10.59976, + "35": 10.61772, + "36": 10.56389, + "37": 10.63614, + "38": 10.53029, + "39": 10.65358, + "40": 10.44072, + "41": 10.49636, + "42": 10.50954, + "43": 10.22362, + "44": 10.30902, + "45": 10.21065, + "46": 10.19943, + "47": 10.41641, + "48": 10.18128, + "49": 9.94311, + "50": 10.21224, + "51": 10.16759, + "52": 10.06895, + "53": 10.30707, + "54": 10.20911, + "55": 10.15688, + "56": 9.91474, + "57": 9.77696, + "58": 10.07417, + "59": 9.86333, + "60": 9.77328, + "61": 9.9292, + "62": 10.17156, + "63": 9.62041, + "64": 9.97113, + "65": 9.21979, + "66": 9.88693, + "67": 9.58363, + "68": 9.94922, + "69": 9.95271, + "70": 9.89312, + "71": 9.77658, + "72": 9.75435, + "73": 9.6497, + "74": 9.1439, + "75": 9.56121, + "76": 9.25111, + "77": 10.17063, + "78": 9.85402, + "79": 9.49965, + "80": 9.53086, + "81": 9.60555, + "82": 9.80179, + "83": 9.43744, + "84": 9.51987, + "85": 9.7196, + "86": 9.18595, + "87": 9.68687, + "88": 9.8443, + "89": 9.70586, + "90": 9.89977, + "91": 9.45029, + "92": 9.45356, + "93": 9.18554, + "94": 8.92968, + "95": 9.59767, + "96": 9.61491, + "97": 9.39084, + "98": 9.75667, + "99": 8.97921, + "100": 9.49001 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 570640384.0, + "2": 570640384.0, + "3": 570640384.0, + "4": 570640384.0, + "5": 570640384.0, + "6": 570640384.0, + "7": 570640384.0, + "8": 570640384.0, + "9": 570640384.0, + "10": 570640384.0, + "11": 570640384.0, + "12": 570640384.0, + "13": 570640384.0, + "14": 570640384.0, + "15": 570640384.0, + "16": 570640384.0, + "17": 852351488.0, + "18": 852351488.0, + "19": 852351488.0, + "20": 852351488.0, + "21": 852351488.0, + "22": 852351488.0, + "23": 852351488.0, + "24": 852351488.0, + "25": 852351488.0, + "26": 852351488.0, + "27": 852351488.0, + "28": 852351488.0, + "29": 852351488.0, + "30": 852351488.0, + "31": 852351488.0, + "32": 852351488.0, + "33": 852351488.0, + "34": 852351488.0, + "35": 852351488.0, + "36": 852351488.0, + "37": 852351488.0, + "38": 852351488.0, + "39": 852351488.0, + "40": 852351488.0, + "41": 852351488.0, + "42": 852351488.0, + "43": 852351488.0, + "44": 852351488.0, + "45": 852351488.0, + "46": 852351488.0, + "47": 852351488.0, + "48": 852351488.0, + "49": 852351488.0, + "50": 852351488.0, + "51": 852351488.0, + "52": 852351488.0, + "53": 852351488.0, + "54": 852351488.0, + "55": 852351488.0, + "56": 852351488.0, + "57": 852351488.0, + "58": 852351488.0, + "59": 852351488.0, + "60": 852351488.0, + "61": 852351488.0, + "62": 852351488.0, + "63": 852351488.0, + "64": 852351488.0, + "65": 852351488.0, + "66": 852351488.0, + "67": 852351488.0, + "68": 852351488.0, + "69": 852351488.0, + "70": 852351488.0, + "71": 852351488.0, + "72": 852351488.0, + "73": 852351488.0, + "74": 852351488.0, + "75": 852351488.0, + "76": 852351488.0, + "77": 852351488.0, + "78": 852351488.0, + "79": 852351488.0, + "80": 852351488.0, + "81": 852351488.0, + "82": 852351488.0, + "83": 852351488.0, + "84": 852351488.0, + "85": 852351488.0, + "86": 852351488.0, + "87": 852351488.0, + "88": 852351488.0, + "89": 852351488.0, + "90": 852351488.0, + "91": 852351488.0, + "92": 852351488.0, + "93": 852351488.0, + "94": 852351488.0, + "95": 852351488.0, + "96": 852351488.0, + "97": 852351488.0, + "98": 852351488.0, + "99": 852351488.0, + "100": 852351488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2394265600.0, + "2": 2394265600.0, + "3": 2394265600.0, + "4": 2394265600.0, + "5": 2394265600.0, + "6": 2394265600.0, + "7": 2394265600.0, + "8": 2394265600.0, + "9": 2394265600.0, + "10": 2394265600.0, + "11": 2394265600.0, + "12": 2394265600.0, + "13": 2394265600.0, + "14": 2394265600.0, + "15": 2394265600.0, + "16": 2394265600.0, + "17": 2394265600.0, + "18": 2675191296.0, + "19": 2675191296.0, + "20": 2675191296.0, + "21": 2675191296.0, + "22": 2675191296.0, + "23": 2675191296.0, + "24": 2675191296.0, + "25": 2675191296.0, + "26": 2675191296.0, + "27": 2675191296.0, + "28": 2675191296.0, + "29": 2675191296.0, + "30": 2675191296.0, + "31": 2675191296.0, + "32": 2675191296.0, + "33": 2675191296.0, + "34": 2675191296.0, + "35": 2675191296.0, + "36": 2675191296.0, + "37": 2675191296.0, + "38": 2675191296.0, + "39": 2675191296.0, + "40": 2675191296.0, + "41": 2675191296.0, + "42": 2675191296.0, + "43": 2675191296.0, + "44": 2675191296.0, + "45": 2675191296.0, + "46": 2675191296.0, + "47": 2675191296.0, + "48": 2675191296.0, + "49": 2675191296.0, + "50": 2675191296.0, + "51": 2675191296.0, + "52": 2675191296.0, + "53": 2675191296.0, + "54": 2675191296.0, + "55": 2675191296.0, + "56": 2675191296.0, + "57": 2675191296.0, + "58": 2675191296.0, + "59": 2675191296.0, + "60": 2675191296.0, + "61": 2675191296.0, + "62": 2675191296.0, + "63": 2675191296.0, + "64": 2675191296.0, + "65": 2675191296.0, + "66": 2675191296.0, + "67": 2675191296.0, + "68": 2675191296.0, + "69": 2675191296.0, + "70": 2675191296.0, + "71": 2675191296.0, + "72": 2675191296.0, + "73": 2675191296.0, + "74": 2675191296.0, + "75": 2675191296.0, + "76": 2675191296.0, + "77": 2675191296.0, + "78": 2675191296.0, + "79": 2675191296.0, + "80": 2675191296.0, + "81": 2675191296.0, + "82": 2675191296.0, + "83": 2675191296.0, + "84": 2675191296.0, + "85": 2675191296.0, + "86": 2675191296.0, + "87": 2675191296.0, + "88": 2675191296.0, + "89": 2675191296.0, + "90": 2675191296.0, + "91": 2675191296.0, + "92": 2675191296.0, + "93": 2675191296.0, + "94": 2675191296.0, + "95": 2675191296.0, + "96": 2675191296.0, + "97": 2675191296.0, + "98": 2675191296.0, + "99": 2675191296.0, + "100": 2675191296.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.94598, + "2": 0.18624, + "3": 0.25859, + "4": 0.15175, + "5": 0.15224, + "6": 0.15109, + "7": 0.15137, + "8": 0.1508, + "9": 0.1517, + "10": 0.15243, + "11": 0.15119, + "12": 0.15082, + "13": 0.15199, + "14": 0.1522, + "15": 0.15257, + "16": 0.15242, + "17": 0.21211, + "18": 0.16275, + "19": 0.15814, + "20": 0.15774, + "21": 0.15784, + "22": 0.15701, + "23": 0.15663, + "24": 0.15702, + "25": 0.15682, + "26": 0.15759, + "27": 0.15671, + "28": 0.15664, + "29": 0.15666, + "30": 0.15733, + "31": 0.15729, + "32": 0.15598, + "33": 0.1563, + "34": 0.15661, + "35": 0.15747, + "36": 0.15621, + "37": 0.15693, + "38": 0.15652, + "39": 0.15686, + "40": 0.15725, + "41": 0.15693, + "42": 0.1567, + "43": 0.15752, + "44": 0.15682, + "45": 0.15685, + "46": 0.15675, + "47": 0.15696, + "48": 0.15702, + "49": 0.15714, + "50": 0.15648, + "51": 0.1656, + "52": 0.15563, + "53": 0.1545, + "54": 0.15554, + "55": 0.15521, + "56": 0.15534, + "57": 0.15463, + "58": 0.15657, + "59": 0.15534, + "60": 0.15488, + "61": 0.15475, + "62": 0.15571, + "63": 0.15676, + "64": 0.15647, + "65": 0.1567, + "66": 0.15541, + "67": 0.15585, + "68": 0.15577, + "69": 0.15571, + "70": 0.15462, + "71": 0.15484, + "72": 0.15532, + "73": 0.15732, + "74": 0.15634, + "75": 0.15545, + "76": 0.1569, + "77": 0.15472, + "78": 0.15486, + "79": 0.15517, + "80": 0.15532, + "81": 0.15621, + "82": 0.15556, + "83": 0.15556, + "84": 0.15546, + "85": 0.15484, + "86": 0.15503, + "87": 0.15496, + "88": 0.15534, + "89": 0.15573, + "90": 0.15449, + "91": 0.15665, + "92": 0.15529, + "93": 0.15627, + "94": 0.15525, + "95": 0.15483, + "96": 0.15509, + "97": 0.15618, + "98": 0.15569, + "99": 0.15482, + "100": 0.15516 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2437.0, + "18": 2405.0, + "19": 2950.0, + "20": 1827.0, + "21": 2154.0, + "22": 2731.0, + "23": 2609.0, + "24": 2290.0, + "25": 2325.0, + "26": 2079.0, + "27": 2138.0, + "28": 2702.0, + "29": 2576.0, + "30": 2528.0, + "31": 1895.0, + "32": 2628.0, + "33": 2325.0, + "34": 1928.0, + "35": 2061.0, + "36": 2153.0, + "37": 2600.0, + "38": 2350.0, + "39": 2997.0, + "40": 2053.0, + "41": 3352.0, + "42": 2497.0, + "43": 2867.0, + "44": 2109.0, + "45": 2490.0, + "46": 2279.0, + "47": 3051.0, + "48": 2527.0, + "49": 1973.0, + "50": 2887.0, + "51": 2310.0, + "52": 2526.0, + "53": 3705.0, + "54": 2888.0, + "55": 2440.0, + "56": 2496.0, + "57": 2338.0, + "58": 3283.0, + "59": 2849.0, + "60": 2893.0, + "61": 2956.0, + "62": 3134.0, + "63": 3275.0, + "64": 3176.0, + "65": 2318.0, + "66": 3857.0, + "67": 2606.0, + "68": 3313.0, + "69": 2826.0, + "70": 3665.0, + "71": 3011.0, + "72": 2693.0, + "73": 3357.0, + "74": 2271.0, + "75": 2955.0, + "76": 3617.0, + "77": 3936.0, + "78": 3951.0, + "79": 4065.0, + "80": 3665.0, + "81": 5191.0, + "82": 3511.0, + "83": 3263.0, + "84": 3876.0, + "85": 4048.0, + "86": 3414.0, + "87": 3980.0, + "88": 3617.0, + "89": 4400.0, + "90": 3695.0, + "91": 2857.0, + "92": 4432.0, + "93": 3494.0, + "94": 4438.0, + "95": 4076.0, + "96": 3948.0, + "97": 4242.0, + "98": 4943.0, + "99": 3861.0, + "100": 3631.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json index a8c7cb012b6..3b0a03dc6ef 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.79219, "5": 10.84727, "10": 10.77729, "15": 10.84106, "20": 10.82889, "25": 10.7666, "30": 10.69961, "35": 10.61845, "40": 10.44051, "45": 10.20859, "50": 10.21168, "55": 10.15675, "60": 9.77265, "65": 9.22128, "70": 9.89371, "75": 9.56098, "80": 9.5311, "85": 9.71911, "90": 9.89983, "95": 9.59785, "100": 9.49009}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 570640384.0, "5": 570640384.0, "10": 570640384.0, "15": 570640384.0, "20": 852351488.0, "25": 852351488.0, "30": 852351488.0, "35": 852351488.0, "40": 852351488.0, "45": 852351488.0, "50": 852351488.0, "55": 852351488.0, "60": 852351488.0, "65": 852351488.0, "70": 852351488.0, "75": 852351488.0, "80": 852351488.0, "85": 852351488.0, "90": 852351488.0, "95": 852351488.0, "100": 852351488.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2393217536.0, "5": 2393218048.0, "10": 2393218048.0, "15": 2393218048.0, "20": 2675191296.0, "25": 2675191296.0, "30": 2675191296.0, "35": 2675191296.0, "40": 2675191296.0, "45": 2675191296.0, "50": 2675191296.0, "55": 2675191296.0, "60": 2675191296.0, "65": 2675191296.0, "70": 2675191296.0, "75": 2675191296.0, "80": 2675191296.0, "85": 2675191296.0, "90": 2675191296.0, "95": 2675191296.0, "100": 2675191296.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 20.32613, "5": 0.16735, "10": 0.16973, "15": 0.16584, "20": 0.17045, "25": 0.17023, "30": 0.16985, "35": 0.17228, "40": 0.17914, "45": 0.17032, "50": 0.17169, "55": 0.16858, "60": 0.16807, "65": 0.16897, "70": 0.16857, "75": 0.16759, "80": 0.16701, "85": 0.16722, "90": 0.16741, "95": 0.16829, "100": 0.16715}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1788.0, "25": 2300.0, "30": 2419.0, "35": 2097.0, "40": 2126.0, "45": 2520.0, "50": 2929.0, "55": 2457.0, "60": 2961.0, "65": 2359.0, "70": 3633.0, "75": 2932.0, "80": 3595.0, "85": 3959.0, "90": 3785.0, "95": 4072.0, "100": 3581.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79449, + "2": 10.80656, + "3": 10.80727, + "4": 10.77389, + "5": 10.84829, + "6": 10.86736, + "7": 10.82922, + "8": 10.81537, + "9": 10.83956, + "10": 10.77652, + "11": 10.90107, + "12": 10.85927, + "13": 10.87396, + "14": 10.89723, + "15": 10.83961, + "16": 10.83508, + "17": 10.82101, + "18": 10.86029, + "19": 10.86558, + "20": 10.82896, + "21": 10.83275, + "22": 10.75286, + "23": 10.88062, + "24": 10.78219, + "25": 10.76607, + "26": 10.79522, + "27": 10.79866, + "28": 10.81697, + "29": 10.82169, + "30": 10.69891, + "31": 10.55698, + "32": 10.75759, + "33": 10.74362, + "34": 10.59976, + "35": 10.61772, + "36": 10.56389, + "37": 10.63614, + "38": 10.53029, + "39": 10.65358, + "40": 10.44072, + "41": 10.49636, + "42": 10.50954, + "43": 10.22362, + "44": 10.30902, + "45": 10.21065, + "46": 10.19943, + "47": 10.41641, + "48": 10.18128, + "49": 9.94311, + "50": 10.21224, + "51": 10.16759, + "52": 10.06895, + "53": 10.30707, + "54": 10.20911, + "55": 10.15688, + "56": 9.91474, + "57": 9.77696, + "58": 10.07417, + "59": 9.86333, + "60": 9.77328, + "61": 9.9292, + "62": 10.17156, + "63": 9.62041, + "64": 9.97113, + "65": 9.21979, + "66": 9.88693, + "67": 9.58363, + "68": 9.94922, + "69": 9.95271, + "70": 9.89312, + "71": 9.77658, + "72": 9.75435, + "73": 9.6497, + "74": 9.1439, + "75": 9.56121, + "76": 9.25111, + "77": 10.17063, + "78": 9.85402, + "79": 9.49965, + "80": 9.53086, + "81": 9.60555, + "82": 9.80179, + "83": 9.43744, + "84": 9.51987, + "85": 9.7196, + "86": 9.18595, + "87": 9.68687, + "88": 9.8443, + "89": 9.70586, + "90": 9.89977, + "91": 9.45029, + "92": 9.45356, + "93": 9.18554, + "94": 8.92968, + "95": 9.59767, + "96": 9.61491, + "97": 9.39084, + "98": 9.75667, + "99": 8.97921, + "100": 9.49001 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 570640384.0, + "2": 570640384.0, + "3": 570640384.0, + "4": 570640384.0, + "5": 570640384.0, + "6": 570640384.0, + "7": 570640384.0, + "8": 570640384.0, + "9": 570640384.0, + "10": 570640384.0, + "11": 570640384.0, + "12": 570640384.0, + "13": 570640384.0, + "14": 570640384.0, + "15": 570640384.0, + "16": 570640384.0, + "17": 852351488.0, + "18": 852351488.0, + "19": 852351488.0, + "20": 852351488.0, + "21": 852351488.0, + "22": 852351488.0, + "23": 852351488.0, + "24": 852351488.0, + "25": 852351488.0, + "26": 852351488.0, + "27": 852351488.0, + "28": 852351488.0, + "29": 852351488.0, + "30": 852351488.0, + "31": 852351488.0, + "32": 852351488.0, + "33": 852351488.0, + "34": 852351488.0, + "35": 852351488.0, + "36": 852351488.0, + "37": 852351488.0, + "38": 852351488.0, + "39": 852351488.0, + "40": 852351488.0, + "41": 852351488.0, + "42": 852351488.0, + "43": 852351488.0, + "44": 852351488.0, + "45": 852351488.0, + "46": 852351488.0, + "47": 852351488.0, + "48": 852351488.0, + "49": 852351488.0, + "50": 852351488.0, + "51": 852351488.0, + "52": 852351488.0, + "53": 852351488.0, + "54": 852351488.0, + "55": 852351488.0, + "56": 852351488.0, + "57": 852351488.0, + "58": 852351488.0, + "59": 852351488.0, + "60": 852351488.0, + "61": 852351488.0, + "62": 852351488.0, + "63": 852351488.0, + "64": 852351488.0, + "65": 852351488.0, + "66": 852351488.0, + "67": 852351488.0, + "68": 852351488.0, + "69": 852351488.0, + "70": 852351488.0, + "71": 852351488.0, + "72": 852351488.0, + "73": 852351488.0, + "74": 852351488.0, + "75": 852351488.0, + "76": 852351488.0, + "77": 852351488.0, + "78": 852351488.0, + "79": 852351488.0, + "80": 852351488.0, + "81": 852351488.0, + "82": 852351488.0, + "83": 852351488.0, + "84": 852351488.0, + "85": 852351488.0, + "86": 852351488.0, + "87": 852351488.0, + "88": 852351488.0, + "89": 852351488.0, + "90": 852351488.0, + "91": 852351488.0, + "92": 852351488.0, + "93": 852351488.0, + "94": 852351488.0, + "95": 852351488.0, + "96": 852351488.0, + "97": 852351488.0, + "98": 852351488.0, + "99": 852351488.0, + "100": 852351488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2393217536.0, + "2": 2393218048.0, + "3": 2393218048.0, + "4": 2394266624.0, + "5": 2394266624.0, + "6": 2394266624.0, + "7": 2394266624.0, + "8": 2394266624.0, + "9": 2394266624.0, + "10": 2394266624.0, + "11": 2394266624.0, + "12": 2394266624.0, + "13": 2394266624.0, + "14": 2394266624.0, + "15": 2394266624.0, + "16": 2394266624.0, + "17": 2394266624.0, + "18": 2675191296.0, + "19": 2675191296.0, + "20": 2675191296.0, + "21": 2675191296.0, + "22": 2675191296.0, + "23": 2675191296.0, + "24": 2675191296.0, + "25": 2675191296.0, + "26": 2675191296.0, + "27": 2675191296.0, + "28": 2675191296.0, + "29": 2675191296.0, + "30": 2675191296.0, + "31": 2675191296.0, + "32": 2675191296.0, + "33": 2675191296.0, + "34": 2675191296.0, + "35": 2675191296.0, + "36": 2675191296.0, + "37": 2675191296.0, + "38": 2675191296.0, + "39": 2675191296.0, + "40": 2675191296.0, + "41": 2675191296.0, + "42": 2675191296.0, + "43": 2675191296.0, + "44": 2675191296.0, + "45": 2675191296.0, + "46": 2675191296.0, + "47": 2675191296.0, + "48": 2675191296.0, + "49": 2675191296.0, + "50": 2675191296.0, + "51": 2675191296.0, + "52": 2675191296.0, + "53": 2675191296.0, + "54": 2675191296.0, + "55": 2675191296.0, + "56": 2675191296.0, + "57": 2675191296.0, + "58": 2675191296.0, + "59": 2675191296.0, + "60": 2675191296.0, + "61": 2675191296.0, + "62": 2675191296.0, + "63": 2675191296.0, + "64": 2675191296.0, + "65": 2675191296.0, + "66": 2675191296.0, + "67": 2675191296.0, + "68": 2675191296.0, + "69": 2675191296.0, + "70": 2675191296.0, + "71": 2675191296.0, + "72": 2675191296.0, + "73": 2675191296.0, + "74": 2675191296.0, + "75": 2675191296.0, + "76": 2675191296.0, + "77": 2675191296.0, + "78": 2675191296.0, + "79": 2675191296.0, + "80": 2675191296.0, + "81": 2675191296.0, + "82": 2675191296.0, + "83": 2675191296.0, + "84": 2675191296.0, + "85": 2675191296.0, + "86": 2675191296.0, + "87": 2675191296.0, + "88": 2675191296.0, + "89": 2675191296.0, + "90": 2675191296.0, + "91": 2675191296.0, + "92": 2675191296.0, + "93": 2675191296.0, + "94": 2675191296.0, + "95": 2675191296.0, + "96": 2675191296.0, + "97": 2675191296.0, + "98": 2675191296.0, + "99": 2675191296.0, + "100": 2675191296.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.58025, + "2": 0.18555, + "3": 0.31194, + "4": 0.1522, + "5": 0.15205, + "6": 0.1496, + "7": 0.14979, + "8": 0.14921, + "9": 0.14957, + "10": 0.15024, + "11": 0.14887, + "12": 0.14852, + "13": 0.14925, + "14": 0.15079, + "15": 0.14925, + "16": 0.14936, + "17": 0.2057, + "18": 0.15996, + "19": 0.15397, + "20": 0.15414, + "21": 0.1543, + "22": 0.15499, + "23": 0.15504, + "24": 0.15679, + "25": 0.15462, + "26": 0.15509, + "27": 0.15394, + "28": 0.15487, + "29": 0.15522, + "30": 0.1553, + "31": 0.15536, + "32": 0.15406, + "33": 0.15461, + "34": 0.1548, + "35": 0.15472, + "36": 0.15413, + "37": 0.1548, + "38": 0.15446, + "39": 0.15545, + "40": 0.15442, + "41": 0.15567, + "42": 0.15413, + "43": 0.15585, + "44": 0.15428, + "45": 0.15497, + "46": 0.15438, + "47": 0.15508, + "48": 0.15481, + "49": 0.15466, + "50": 0.15476, + "51": 0.16245, + "52": 0.15411, + "53": 0.15376, + "54": 0.15405, + "55": 0.15375, + "56": 0.15402, + "57": 0.15434, + "58": 0.15404, + "59": 0.15454, + "60": 0.15434, + "61": 0.15384, + "62": 0.15505, + "63": 0.15431, + "64": 0.15388, + "65": 0.1547, + "66": 0.15453, + "67": 0.15364, + "68": 0.15388, + "69": 0.15362, + "70": 0.15366, + "71": 0.15425, + "72": 0.15393, + "73": 0.15476, + "74": 0.15414, + "75": 0.15415, + "76": 0.1535, + "77": 0.15481, + "78": 0.1541, + "79": 0.15382, + "80": 0.15363, + "81": 0.15386, + "82": 0.18555, + "83": 0.15422, + "84": 0.15393, + "85": 0.15462, + "86": 0.15512, + "87": 0.15391, + "88": 0.15431, + "89": 0.15431, + "90": 0.15521, + "91": 0.15475, + "92": 0.154, + "93": 0.15414, + "94": 0.15426, + "95": 0.15422, + "96": 0.15393, + "97": 0.15497, + "98": 0.1538, + "99": 0.15481, + "100": 0.15442 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2437.0, + "18": 2405.0, + "19": 2950.0, + "20": 1827.0, + "21": 2154.0, + "22": 2731.0, + "23": 2609.0, + "24": 2290.0, + "25": 2325.0, + "26": 2079.0, + "27": 2138.0, + "28": 2702.0, + "29": 2576.0, + "30": 2528.0, + "31": 1895.0, + "32": 2628.0, + "33": 2325.0, + "34": 1928.0, + "35": 2061.0, + "36": 2153.0, + "37": 2600.0, + "38": 2350.0, + "39": 2997.0, + "40": 2053.0, + "41": 3352.0, + "42": 2497.0, + "43": 2867.0, + "44": 2109.0, + "45": 2490.0, + "46": 2279.0, + "47": 3051.0, + "48": 2527.0, + "49": 1973.0, + "50": 2887.0, + "51": 2310.0, + "52": 2526.0, + "53": 3705.0, + "54": 2888.0, + "55": 2440.0, + "56": 2496.0, + "57": 2338.0, + "58": 3283.0, + "59": 2849.0, + "60": 2893.0, + "61": 2956.0, + "62": 3134.0, + "63": 3275.0, + "64": 3176.0, + "65": 2318.0, + "66": 3857.0, + "67": 2606.0, + "68": 3313.0, + "69": 2826.0, + "70": 3665.0, + "71": 3011.0, + "72": 2693.0, + "73": 3357.0, + "74": 2271.0, + "75": 2955.0, + "76": 3617.0, + "77": 3936.0, + "78": 3951.0, + "79": 4065.0, + "80": 3665.0, + "81": 5191.0, + "82": 3511.0, + "83": 3263.0, + "84": 3876.0, + "85": 4048.0, + "86": 3414.0, + "87": 3980.0, + "88": 3617.0, + "89": 4400.0, + "90": 3695.0, + "91": 2857.0, + "92": 4432.0, + "93": 3494.0, + "94": 4438.0, + "95": 4076.0, + "96": 3948.0, + "97": 4242.0, + "98": 4943.0, + "99": 3861.0, + "100": 3631.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json new file mode 100644 index 00000000000..646419f3d75 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465367040.0, + "4": 1465367040.0, + "5": 1465367040.0, + "6": 1465367040.0, + "7": 1465368064.0, + "8": 1465368064.0, + "9": 1465368064.0, + "10": 1465368064.0, + "11": 1465368064.0, + "12": 1465368064.0, + "13": 1465368064.0, + "14": 1465368064.0, + "15": 1465368064.0, + "16": 1465368064.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.93622, + "2": 0.31349, + "3": 0.28163, + "4": 0.27921, + "5": 0.27918, + "6": 0.27813, + "7": 0.27823, + "8": 0.27889, + "9": 0.27839, + "10": 0.2785, + "11": 0.27567, + "12": 0.27474, + "13": 0.2746, + "14": 0.27424, + "15": 0.27387, + "16": 0.4724, + "17": 0.27409, + "18": 0.28601, + "19": 0.28651, + "20": 0.28586, + "21": 0.29036, + "22": 0.27749, + "23": 0.29042, + "24": 0.29143, + "25": 0.29191, + "26": 0.29136, + "27": 0.29223, + "28": 0.29253, + "29": 0.29127, + "30": 0.29158, + "31": 0.29283, + "32": 0.29224, + "33": 0.29404, + "34": 0.29214, + "35": 0.29125, + "36": 0.29141, + "37": 0.29119, + "38": 0.29131, + "39": 0.29195, + "40": 0.29134, + "41": 0.29104, + "42": 0.2909, + "43": 0.29202, + "44": 0.29149, + "45": 0.29285, + "46": 0.29117, + "47": 0.29278, + "48": 0.29328, + "49": 0.2921, + "50": 0.29144, + "51": 0.2946, + "52": 0.29041, + "53": 0.28952, + "54": 0.28982, + "55": 0.28907, + "56": 0.28812, + "57": 0.2882, + "58": 0.29098, + "59": 0.29394, + "60": 0.29381, + "61": 0.29366, + "62": 0.29495, + "63": 0.29248, + "64": 0.29429, + "65": 0.29369, + "66": 0.29315, + "67": 0.29284, + "68": 0.2941, + "69": 0.29349, + "70": 0.29458, + "71": 0.29385, + "72": 0.29691, + "73": 0.29214, + "74": 0.292, + "75": 0.29157, + "76": 0.29169, + "77": 0.29162, + "78": 0.2922, + "79": 0.29243, + "80": 0.29113, + "81": 0.2927, + "82": 0.29109, + "83": 0.29115, + "84": 0.29064, + "85": 0.29302, + "86": 0.29203, + "87": 0.29135, + "88": 0.29306, + "89": 0.29406, + "90": 0.29439, + "91": 0.29348, + "92": 0.30393, + "93": 0.29287, + "94": 0.29232, + "95": 0.29711, + "96": 0.29103, + "97": 0.29472, + "98": 0.29334, + "99": 0.29389, + "100": 0.29184 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json index c8abc0e775c..9f83249318a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86104, "5": 10.88371, "10": 10.84263, "15": 10.87935, "20": 10.87405, "25": 10.82867, "30": 10.77191, "35": 10.67622, "40": 10.517, "45": 10.28436, "50": 10.27861, "55": 10.20112, "60": 9.83306, "65": 9.26979, "70": 9.92662, "75": 9.61385, "80": 9.56419, "85": 9.74319, "90": 9.92148, "95": 9.6163, "100": 9.5087}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 284527616.0, "5": 284527616.0, "10": 284527616.0, "15": 284527616.0, "20": 416513536.0, "25": 416513536.0, "30": 416513536.0, "35": 416513536.0, "40": 416513536.0, "45": 416513536.0, "50": 416513536.0, "55": 416513536.0, "60": 416513536.0, "65": 416513536.0, "70": 416513536.0, "75": 416513536.0, "80": 416513536.0, "85": 416513536.0, "90": 416513536.0, "95": 416513536.0, "100": 416513536.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1465368064.0, "5": 1465368576.0, "10": 1465368576.0, "15": 1465368576.0, "20": 1596305408.0, "25": 1596305920.0, "30": 1596305920.0, "35": 1596305920.0, "40": 1596305920.0, "45": 1596305920.0, "50": 1596305920.0, "55": 1596305920.0, "60": 1596305920.0, "65": 1596305920.0, "70": 1596305920.0, "75": 1596305920.0, "80": 1596305920.0, "85": 1596305920.0, "90": 1596305920.0, "95": 1596305920.0, "100": 1596305920.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 7.32922, "5": 0.3111, "10": 0.3117, "15": 0.30486, "20": 0.3229, "25": 0.31791, "30": 0.31805, "35": 0.32249, "40": 0.32048, "45": 0.32173, "50": 0.31691, "55": 0.31145, "60": 0.31713, "65": 0.31381, "70": 0.31308, "75": 0.31799, "80": 0.31492, "85": 0.31756, "90": 0.31282, "95": 0.41456, "100": 0.30993}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1777.0, "25": 2124.0, "30": 2262.0, "35": 1763.0, "40": 1875.0, "45": 2434.0, "50": 2634.0, "55": 2383.0, "60": 2832.0, "65": 2223.0, "70": 3836.0, "75": 2913.0, "80": 3398.0, "85": 3618.0, "90": 3676.0, "95": 4082.0, "100": 3502.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367552.0, + "3": 1465367552.0, + "4": 1465367552.0, + "5": 1465367552.0, + "6": 1465367552.0, + "7": 1465367552.0, + "8": 1465367552.0, + "9": 1465367552.0, + "10": 1465367552.0, + "11": 1465367552.0, + "12": 1465367552.0, + "13": 1465367552.0, + "14": 1465367552.0, + "15": 1465367552.0, + "16": 1465367552.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.81628, + "2": 0.32142, + "3": 0.27555, + "4": 0.28299, + "5": 0.28901, + "6": 0.28043, + "7": 0.29138, + "8": 0.30944, + "9": 0.28461, + "10": 0.28789, + "11": 0.28709, + "12": 0.29186, + "13": 0.29114, + "14": 0.29464, + "15": 0.31626, + "16": 0.48847, + "17": 0.28436, + "18": 0.30264, + "19": 0.29287, + "20": 0.30599, + "21": 0.29335, + "22": 0.27957, + "23": 0.29491, + "24": 0.29371, + "25": 0.29398, + "26": 0.29344, + "27": 0.29457, + "28": 0.29449, + "29": 0.29412, + "30": 0.29337, + "31": 0.29404, + "32": 0.29391, + "33": 0.29483, + "34": 0.29389, + "35": 0.29433, + "36": 0.29449, + "37": 0.29463, + "38": 0.29428, + "39": 0.29385, + "40": 0.29379, + "41": 0.29345, + "42": 0.29404, + "43": 0.29413, + "44": 0.29357, + "45": 0.29308, + "46": 0.29302, + "47": 0.29311, + "48": 0.29341, + "49": 0.2946, + "50": 0.29365, + "51": 0.29978, + "52": 0.31599, + "53": 0.29361, + "54": 0.29341, + "55": 0.29321, + "56": 0.29262, + "57": 0.29474, + "58": 0.29427, + "59": 0.29281, + "60": 0.29314, + "61": 0.29219, + "62": 0.29346, + "63": 0.29348, + "64": 0.30211, + "65": 0.29324, + "66": 0.29357, + "67": 0.29314, + "68": 0.29229, + "69": 0.30197, + "70": 0.29329, + "71": 0.30206, + "72": 0.29435, + "73": 0.29495, + "74": 0.2943, + "75": 0.29926, + "76": 0.29332, + "77": 0.29464, + "78": 0.29342, + "79": 0.29434, + "80": 0.29439, + "81": 0.29391, + "82": 0.29436, + "83": 0.29426, + "84": 0.29408, + "85": 0.29452, + "86": 0.29406, + "87": 0.29421, + "88": 0.29373, + "89": 0.29437, + "90": 0.29425, + "91": 0.29383, + "92": 0.2933, + "93": 0.29369, + "94": 0.2937, + "95": 0.29465, + "96": 0.29439, + "97": 0.29435, + "98": 0.2952, + "99": 0.29361, + "100": 0.2936 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..a7feec6d207 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465368576.0, + "3": 1465368576.0, + "4": 1465368576.0, + "5": 1465368576.0, + "6": 1465368576.0, + "7": 1465368576.0, + "8": 1465368576.0, + "9": 1465368576.0, + "10": 1465368576.0, + "11": 1465368576.0, + "12": 1465368576.0, + "13": 1465368576.0, + "14": 1465368576.0, + "15": 1465368576.0, + "16": 1465368576.0, + "17": 1597092352.0, + "18": 1597484544.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.79491, + "2": 0.31619, + "3": 0.28095, + "4": 0.28167, + "5": 0.28111, + "6": 0.28053, + "7": 0.27397, + "8": 0.27382, + "9": 0.2729, + "10": 0.28341, + "11": 0.27301, + "12": 0.27739, + "13": 0.27433, + "14": 0.27589, + "15": 0.27377, + "16": 0.45757, + "17": 0.28453, + "18": 0.29428, + "19": 0.29061, + "20": 0.28748, + "21": 0.28947, + "22": 0.27305, + "23": 0.28802, + "24": 0.29052, + "25": 0.28857, + "26": 0.28938, + "27": 0.28789, + "28": 0.29629, + "29": 0.29633, + "30": 0.29452, + "31": 0.29612, + "32": 0.29638, + "33": 0.29802, + "34": 0.29681, + "35": 0.29807, + "36": 0.29648, + "37": 0.29732, + "38": 0.29691, + "39": 0.29854, + "40": 0.29738, + "41": 0.29825, + "42": 0.29748, + "43": 0.29734, + "44": 0.29746, + "45": 0.29836, + "46": 0.29887, + "47": 0.29739, + "48": 0.29926, + "49": 0.29788, + "50": 0.29895, + "51": 0.29828, + "52": 0.29254, + "53": 0.29242, + "54": 0.29234, + "55": 0.29132, + "56": 0.29158, + "57": 0.29153, + "58": 0.29113, + "59": 0.29239, + "60": 0.29622, + "61": 0.29477, + "62": 0.29358, + "63": 0.29465, + "64": 0.29445, + "65": 0.29533, + "66": 0.29531, + "67": 0.29354, + "68": 0.29434, + "69": 0.29379, + "70": 0.29468, + "71": 0.29423, + "72": 0.29484, + "73": 0.29584, + "74": 0.2949, + "75": 0.29561, + "76": 0.29586, + "77": 0.29535, + "78": 0.29491, + "79": 0.29562, + "80": 0.29515, + "81": 0.2951, + "82": 0.29484, + "83": 0.2928, + "84": 0.29076, + "85": 0.29104, + "86": 0.29208, + "87": 0.29116, + "88": 0.29384, + "89": 0.29252, + "90": 0.29125, + "91": 0.29184, + "92": 0.29214, + "93": 0.29151, + "94": 0.29183, + "95": 0.29202, + "96": 0.29138, + "97": 0.29213, + "98": 0.2924, + "99": 0.29154, + "100": 0.29129 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json index f9728255608..0eef09cf2c1 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86104, "5": 10.88371, "10": 10.84263, "15": 10.87935, "20": 10.87405, "25": 10.82867, "30": 10.77191, "35": 10.67622, "40": 10.517, "45": 10.28436, "50": 10.27861, "55": 10.20112, "60": 9.83306, "65": 9.26979, "70": 9.92662, "75": 9.61385, "80": 9.56419, "85": 9.74319, "90": 9.92148, "95": 9.6163, "100": 9.5087}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 284527616.0, "5": 284527616.0, "10": 284527616.0, "15": 284527616.0, "20": 416513536.0, "25": 416513536.0, "30": 416513536.0, "35": 416513536.0, "40": 416513536.0, "45": 416513536.0, "50": 416513536.0, "55": 416513536.0, "60": 416513536.0, "65": 416513536.0, "70": 416513536.0, "75": 416513536.0, "80": 416513536.0, "85": 416513536.0, "90": 416513536.0, "95": 416513536.0, "100": 416513536.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1465368064.0, "5": 1465368576.0, "10": 1465368576.0, "15": 1465368576.0, "20": 1596305920.0, "25": 1596305920.0, "30": 1596305920.0, "35": 1596305920.0, "40": 1596305920.0, "45": 1596305920.0, "50": 1596305920.0, "55": 1596305920.0, "60": 1596305920.0, "65": 1596305920.0, "70": 1596305920.0, "75": 1596305920.0, "80": 1596305920.0, "85": 1596305920.0, "90": 1596305920.0, "95": 1596305920.0, "100": 1596305920.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 8.79594, "5": 0.30836, "10": 0.30237, "15": 0.30482, "20": 0.31834, "25": 0.31761, "30": 0.31732, "35": 0.31511, "40": 0.31755, "45": 0.31516, "50": 0.31689, "55": 0.31106, "60": 0.31323, "65": 0.31051, "70": 0.31046, "75": 0.31201, "80": 0.30466, "85": 0.3063, "90": 0.30526, "95": 0.30371, "100": 0.30515}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1777.0, "25": 2124.0, "30": 2262.0, "35": 1763.0, "40": 1875.0, "45": 2434.0, "50": 2634.0, "55": 2383.0, "60": 2832.0, "65": 2223.0, "70": 3836.0, "75": 2913.0, "80": 3398.0, "85": 3618.0, "90": 3676.0, "95": 4082.0, "100": 3502.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465368064.0, + "4": 1465368576.0, + "5": 1465368576.0, + "6": 1465368576.0, + "7": 1465368576.0, + "8": 1465368576.0, + "9": 1465368576.0, + "10": 1465368576.0, + "11": 1465368576.0, + "12": 1465368576.0, + "13": 1465368576.0, + "14": 1465368576.0, + "15": 1465368576.0, + "16": 1465368576.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.02782, + "2": 0.31435, + "3": 0.27957, + "4": 0.27933, + "5": 0.27866, + "6": 0.27855, + "7": 0.2779, + "8": 0.27621, + "9": 0.27704, + "10": 0.27611, + "11": 0.27501, + "12": 0.27489, + "13": 0.27468, + "14": 0.27386, + "15": 0.27315, + "16": 0.41595, + "17": 0.27523, + "18": 0.28979, + "19": 0.28871, + "20": 0.2888, + "21": 0.28867, + "22": 0.27653, + "23": 0.29205, + "24": 0.29078, + "25": 0.29104, + "26": 0.29087, + "27": 0.28794, + "28": 0.28784, + "29": 0.28659, + "30": 0.28669, + "31": 0.28638, + "32": 0.2878, + "33": 0.28717, + "34": 0.28616, + "35": 0.28626, + "36": 0.28648, + "37": 0.28977, + "38": 0.28615, + "39": 0.2864, + "40": 0.28588, + "41": 0.28749, + "42": 0.28735, + "43": 0.28605, + "44": 0.28798, + "45": 0.2882, + "46": 0.28727, + "47": 0.28616, + "48": 0.28603, + "49": 0.2876, + "50": 0.29155, + "51": 0.30309, + "52": 0.29889, + "53": 0.29736, + "54": 0.29772, + "55": 0.29611, + "56": 0.29565, + "57": 0.29413, + "58": 0.29391, + "59": 0.29344, + "60": 0.29428, + "61": 0.29695, + "62": 0.29282, + "63": 0.29418, + "64": 0.29352, + "65": 0.29274, + "66": 0.29449, + "67": 0.29627, + "68": 0.29636, + "69": 0.29393, + "70": 0.28967, + "71": 0.28925, + "72": 0.28962, + "73": 0.28944, + "74": 0.28948, + "75": 0.28996, + "76": 0.28938, + "77": 0.28855, + "78": 0.28891, + "79": 0.28905, + "80": 0.28968, + "81": 0.28873, + "82": 0.28966, + "83": 0.2884, + "84": 0.28842, + "85": 0.29077, + "86": 0.28927, + "87": 0.28888, + "88": 0.28909, + "89": 0.28807, + "90": 0.28887, + "91": 0.28894, + "92": 0.28908, + "93": 0.28985, + "94": 0.289, + "95": 0.28861, + "96": 0.28831, + "97": 0.2877, + "98": 0.29019, + "99": 0.28839, + "100": 0.2881 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgx_h100.json index 024ac59e078..12a9b70df83 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgx_h100.json @@ -174,5 +174,5 @@ -0.7878209352493286 ] }, - "throughput": 104.98559493782837 + "throughput": [104.98559493782837, 104.98559493782837] } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgx_h100.json index 18a5978a127..8e07dfee229 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgx_h100.json @@ -174,5 +174,5 @@ -0.7878209352493286 ] }, - "throughput": 79.88988160240554 + "throughput": [79.88988160240554, 79.88988160240554] } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json index 3ba402cbcae..dd8b08e446f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json @@ -2695,5 +2695,5 @@ -0.00032085992279462516 ] }, - "throughput": 107.66332959870442 + "throughput": [107.66332959870442, 107.66332959870442] } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json index fe938d51e4f..6ef98105cbd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgx_h100.json @@ -157,5 +157,5 @@ -0.0585334412753582 ] }, - "throughput": 13.93210545115292 + "throughput": [13.93210545115292, 13.93210545115292] } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..d547773495d --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81189, + "2": 10.82837, + "3": 10.81793, + "4": 10.80617, + "5": 10.84927, + "6": 10.86922, + "7": 10.84362, + "8": 10.83419, + "9": 10.84988, + "10": 10.78408, + "11": 10.88038, + "12": 10.85362, + "13": 10.86619, + "14": 10.88223, + "15": 10.82311, + "16": 10.81052, + "17": 10.78202, + "18": 10.81541, + "19": 10.81756, + "20": 10.74584, + "21": 10.72627, + "22": 10.60022, + "23": 10.74769, + "24": 10.6365, + "25": 10.59453, + "26": 10.64799, + "27": 10.66378, + "28": 10.64562, + "29": 10.65242, + "30": 10.44192, + "31": 10.21776, + "32": 10.52786, + "33": 10.52222, + "34": 10.30567, + "35": 10.35142, + "36": 10.32502, + "37": 10.42585, + "38": 10.28522, + "39": 10.47826, + "40": 10.19414, + "41": 10.22953, + "42": 10.29398, + "43": 9.95703, + "44": 10.06609, + "45": 9.95939, + "46": 9.92711, + "47": 10.22774, + "48": 9.96376, + "49": 9.65724, + "50": 10.01644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31381.0, + "2": 33910.0, + "3": 33570.0, + "4": 31384.0, + "5": 35903.0, + "6": 37599.0, + "7": 35525.0, + "8": 31635.0, + "9": 34509.0, + "10": 30410.0, + "11": 37757.0, + "12": 35967.0, + "13": 36886.0, + "14": 38227.0, + "15": 35788.0, + "16": 36247.0, + "17": 35378.0, + "18": 35284.0, + "19": 36213.0, + "20": 33124.0, + "21": 33424.0, + "22": 31395.0, + "23": 38414.0, + "24": 32277.0, + "25": 31634.0, + "26": 35264.0, + "27": 35935.0, + "28": 37638.0, + "29": 38145.0, + "30": 33261.0, + "31": 30972.0, + "32": 36646.0, + "33": 38318.0, + "34": 33243.0, + "35": 34813.0, + "36": 35446.0, + "37": 38278.0, + "38": 35590.0, + "39": 38681.0, + "40": 36027.0, + "41": 36509.0, + "42": 37301.0, + "43": 34597.0, + "44": 34024.0, + "45": 35938.0, + "46": 37229.0, + "47": 40862.0, + "48": 35537.0, + "49": 35095.0, + "50": 39711.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 880441344.0, + "2": 880437760.0, + "3": 880442368.0, + "4": 880448512.0, + "5": 880440832.0, + "6": 880441344.0, + "7": 880445440.0, + "8": 880442880.0, + "9": 880440320.0, + "10": 880443392.0, + "11": 880442368.0, + "12": 880439808.0, + "13": 880439808.0, + "14": 880443904.0, + "15": 880443904.0, + "16": 880441856.0, + "17": 880448512.0, + "18": 880437760.0, + "19": 880440320.0, + "20": 880442880.0, + "21": 880445440.0, + "22": 880450048.0, + "23": 880449024.0, + "24": 880440320.0, + "25": 880440320.0, + "26": 880445952.0, + "27": 880441344.0, + "28": 880439808.0, + "29": 880443392.0, + "30": 880441344.0, + "31": 880454656.0, + "32": 880444928.0, + "33": 880440320.0, + "34": 880440832.0, + "35": 880446976.0, + "36": 880441344.0, + "37": 880442880.0, + "38": 880439808.0, + "39": 880441856.0, + "40": 880442880.0, + "41": 880445952.0, + "42": 880441344.0, + "43": 880444416.0, + "44": 880439808.0, + "45": 880440832.0, + "46": 880447488.0, + "47": 880440320.0, + "48": 880441856.0, + "49": 880444416.0, + "50": 880442368.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2909851648.0, + "2": 3151821824.0, + "3": 3152806912.0, + "4": 3156619264.0, + "5": 3156619264.0, + "6": 3156619264.0, + "7": 3156619264.0, + "8": 3156619264.0, + "9": 3156619264.0, + "10": 3156619264.0, + "11": 3156619264.0, + "12": 3156619264.0, + "13": 3156619264.0, + "14": 3156619264.0, + "15": 3156619264.0, + "16": 3156619264.0, + "17": 3156619264.0, + "18": 3156619264.0, + "19": 3156619264.0, + "20": 3156619264.0, + "21": 3156619264.0, + "22": 3156619264.0, + "23": 3156619264.0, + "24": 3156619264.0, + "25": 3156619264.0, + "26": 3156619264.0, + "27": 3156619264.0, + "28": 3156619264.0, + "29": 3156619264.0, + "30": 3156619264.0, + "31": 3167942656.0, + "32": 3167942656.0, + "33": 3167942656.0, + "34": 3167942656.0, + "35": 3167942656.0, + "36": 3167942656.0, + "37": 3167942656.0, + "38": 3167942656.0, + "39": 3167942656.0, + "40": 3167942656.0, + "41": 3167942656.0, + "42": 3167942656.0, + "43": 3167942656.0, + "44": 3167942656.0, + "45": 3167942656.0, + "46": 3167942656.0, + "47": 3167942656.0, + "48": 3167942656.0, + "49": 3167942656.0, + "50": 3167942656.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.71086, + "2": 0.34876, + "3": 0.30494, + "4": 0.2983, + "5": 0.31094, + "6": 0.29581, + "7": 0.29535, + "8": 0.29392, + "9": 0.31226, + "10": 0.29284, + "11": 0.29297, + "12": 0.29417, + "13": 0.29411, + "14": 0.29617, + "15": 0.30137, + "16": 0.29884, + "17": 0.29723, + "18": 0.2965, + "19": 0.29582, + "20": 0.29827, + "21": 0.29422, + "22": 0.29464, + "23": 0.29372, + "24": 0.29476, + "25": 0.29448, + "26": 0.29512, + "27": 0.29393, + "28": 0.2949, + "29": 0.29363, + "30": 0.29407, + "31": 0.29478, + "32": 0.2959, + "33": 0.29578, + "34": 0.29539, + "35": 0.295, + "36": 0.29765, + "37": 0.29552, + "38": 0.2962, + "39": 0.29593, + "40": 0.29517, + "41": 0.29558, + "42": 0.2974, + "43": 0.29483, + "44": 0.29357, + "45": 0.29143, + "46": 0.29287, + "47": 0.28977, + "48": 0.30711, + "49": 0.29644, + "50": 0.29278 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json index 63ffa2aa706..f91ad30ed3a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.8028, "5": 10.85317, "10": 10.79253, "15": 10.81762, "20": 10.74263, "25": 10.58726, "30": 10.43425, "35": 10.35057, "40": 10.18634, "45": 9.95891, "50": 10.0194}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 31321.0, "5": 36637.0, "10": 30318.0, "15": 35096.0, "20": 33013.0, "25": 30677.0, "30": 32775.0, "35": 34420.0, "40": 36074.0, "45": 35624.0, "50": 39159.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 863505920.0, "5": 863502848.0, "10": 863507456.0, "15": 863503872.0, "20": 863504896.0, "25": 863506944.0, "30": 863508992.0, "35": 863511040.0, "40": 863508480.0, "45": 863503872.0, "50": 863505920.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2741182976.0, "5": 2984047104.0, "10": 2984047104.0, "15": 2984047104.0, "20": 2985508864.0, "25": 2985508864.0, "30": 2985508864.0, "35": 2986774016.0, "40": 2988336640.0, "45": 2988336640.0, "50": 2990856704.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 30.77131, "5": 0.27562, "10": 0.2823, "15": 0.2732, "20": 0.2737, "25": 0.28195, "30": 0.27083, "35": 0.27437, "40": 0.26787, "45": 0.26819, "50": 0.28362}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81189, + "2": 10.82837, + "3": 10.81793, + "4": 10.80617, + "5": 10.84927, + "6": 10.86922, + "7": 10.84362, + "8": 10.83419, + "9": 10.84988, + "10": 10.78408, + "11": 10.88038, + "12": 10.85362, + "13": 10.86619, + "14": 10.88223, + "15": 10.82311, + "16": 10.81052, + "17": 10.78202, + "18": 10.81541, + "19": 10.81756, + "20": 10.74584, + "21": 10.72627, + "22": 10.60022, + "23": 10.74769, + "24": 10.6365, + "25": 10.59453, + "26": 10.64799, + "27": 10.66378, + "28": 10.64562, + "29": 10.65242, + "30": 10.44192, + "31": 10.21776, + "32": 10.52786, + "33": 10.52222, + "34": 10.30567, + "35": 10.35142, + "36": 10.32502, + "37": 10.42585, + "38": 10.28522, + "39": 10.47826, + "40": 10.19414, + "41": 10.22953, + "42": 10.29398, + "43": 9.95703, + "44": 10.06609, + "45": 9.95939, + "46": 9.92711, + "47": 10.22774, + "48": 9.96376, + "49": 9.65724, + "50": 10.01644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31381.0, + "2": 33910.0, + "3": 33570.0, + "4": 31384.0, + "5": 35903.0, + "6": 37599.0, + "7": 35525.0, + "8": 31635.0, + "9": 34509.0, + "10": 30410.0, + "11": 37757.0, + "12": 35967.0, + "13": 36886.0, + "14": 38227.0, + "15": 35788.0, + "16": 36247.0, + "17": 35378.0, + "18": 35284.0, + "19": 36213.0, + "20": 33124.0, + "21": 33424.0, + "22": 31395.0, + "23": 38414.0, + "24": 32277.0, + "25": 31634.0, + "26": 35264.0, + "27": 35935.0, + "28": 37638.0, + "29": 38145.0, + "30": 33261.0, + "31": 30972.0, + "32": 36646.0, + "33": 38318.0, + "34": 33243.0, + "35": 34813.0, + "36": 35446.0, + "37": 38278.0, + "38": 35590.0, + "39": 38681.0, + "40": 36027.0, + "41": 36509.0, + "42": 37301.0, + "43": 34597.0, + "44": 34024.0, + "45": 35938.0, + "46": 37229.0, + "47": 40862.0, + "48": 35537.0, + "49": 35095.0, + "50": 39711.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 880441344.0, + "2": 880437760.0, + "3": 880442368.0, + "4": 880448512.0, + "5": 880440832.0, + "6": 880441344.0, + "7": 880445440.0, + "8": 880442880.0, + "9": 880440320.0, + "10": 880443392.0, + "11": 880442368.0, + "12": 880439808.0, + "13": 880439808.0, + "14": 880443904.0, + "15": 880443904.0, + "16": 880441856.0, + "17": 880448512.0, + "18": 880437760.0, + "19": 880440320.0, + "20": 880442880.0, + "21": 880445440.0, + "22": 880450048.0, + "23": 880449024.0, + "24": 880440320.0, + "25": 880440320.0, + "26": 880445952.0, + "27": 880441344.0, + "28": 880439808.0, + "29": 880443392.0, + "30": 880441344.0, + "31": 880454656.0, + "32": 880444928.0, + "33": 880440320.0, + "34": 880440832.0, + "35": 880446976.0, + "36": 880441344.0, + "37": 880442880.0, + "38": 880439808.0, + "39": 880441856.0, + "40": 880442880.0, + "41": 880445952.0, + "42": 880441344.0, + "43": 880444416.0, + "44": 880439808.0, + "45": 880440832.0, + "46": 880447488.0, + "47": 880440320.0, + "48": 880441856.0, + "49": 880444416.0, + "50": 880442368.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2910859264.0, + "2": 3151821824.0, + "3": 3152806912.0, + "4": 3156619264.0, + "5": 3156619264.0, + "6": 3156619264.0, + "7": 3156619264.0, + "8": 3156619264.0, + "9": 3156619264.0, + "10": 3156619264.0, + "11": 3156619264.0, + "12": 3156619264.0, + "13": 3156619264.0, + "14": 3156619264.0, + "15": 3156619264.0, + "16": 3156619264.0, + "17": 3156619264.0, + "18": 3156619264.0, + "19": 3156619264.0, + "20": 3156619264.0, + "21": 3156619264.0, + "22": 3156619264.0, + "23": 3156619264.0, + "24": 3156619264.0, + "25": 3156619264.0, + "26": 3156619264.0, + "27": 3156619264.0, + "28": 3156619264.0, + "29": 3156619264.0, + "30": 3156619264.0, + "31": 3167942656.0, + "32": 3167942656.0, + "33": 3167942656.0, + "34": 3167942656.0, + "35": 3167942656.0, + "36": 3167942656.0, + "37": 3167942656.0, + "38": 3167942656.0, + "39": 3167942656.0, + "40": 3167942656.0, + "41": 3167942656.0, + "42": 3167942656.0, + "43": 3167942656.0, + "44": 3167942656.0, + "45": 3167942656.0, + "46": 3167942656.0, + "47": 3167942656.0, + "48": 3167942656.0, + "49": 3167942656.0, + "50": 3167942656.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.90142, + "2": 0.35609, + "3": 0.29589, + "4": 0.29327, + "5": 0.29594, + "6": 0.293, + "7": 0.29087, + "8": 0.29178, + "9": 0.29184, + "10": 0.29303, + "11": 0.29381, + "12": 0.29249, + "13": 0.2936, + "14": 0.29671, + "15": 0.29969, + "16": 0.30214, + "17": 0.29463, + "18": 0.30986, + "19": 0.29429, + "20": 0.29497, + "21": 0.29609, + "22": 0.29421, + "23": 0.2931, + "24": 0.29341, + "25": 0.29443, + "26": 0.28879, + "27": 0.28844, + "28": 0.28873, + "29": 0.28741, + "30": 0.28737, + "31": 0.28905, + "32": 0.28701, + "33": 0.28706, + "34": 0.28739, + "35": 0.28701, + "36": 0.28751, + "37": 0.28826, + "38": 0.28792, + "39": 0.28663, + "40": 0.28805, + "41": 0.28776, + "42": 0.28855, + "43": 0.28777, + "44": 0.28801, + "45": 0.2885, + "46": 0.28907, + "47": 0.28755, + "48": 0.28719, + "49": 0.28878, + "50": 0.28677 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json new file mode 100644 index 00000000000..00b01bf0048 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.78519, + "2": 10.81141, + "3": 10.79553, + "4": 10.78363, + "5": 10.83004, + "6": 10.8417, + "7": 10.81276, + "8": 10.81576, + "9": 10.82408, + "10": 10.75834, + "11": 10.86449, + "12": 10.83694, + "13": 10.84627, + "14": 10.85637, + "15": 10.79099, + "16": 10.78206, + "17": 10.73911, + "18": 10.77969, + "19": 10.77703, + "20": 10.69044, + "21": 10.66086, + "22": 10.51537, + "23": 10.68288, + "24": 10.54469, + "25": 10.49929, + "26": 10.5582, + "27": 10.57541, + "28": 10.54334, + "29": 10.55412, + "30": 10.31857, + "31": 10.06911, + "32": 10.42007, + "33": 10.42435, + "34": 10.16897, + "35": 10.22944, + "36": 10.19358, + "37": 10.29976, + "38": 10.1465, + "39": 10.37282, + "40": 10.04287, + "41": 10.09714, + "42": 10.17339, + "43": 9.79317, + "44": 9.91871, + "45": 9.79389, + "46": 9.77586, + "47": 10.10721, + "48": 9.81704, + "49": 9.48887, + "50": 9.87988 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 26764.0, + "2": 29274.0, + "3": 28827.0, + "4": 27434.0, + "5": 31796.0, + "6": 32589.0, + "7": 31434.0, + "8": 27373.0, + "9": 30359.0, + "10": 25775.0, + "11": 33420.0, + "12": 30835.0, + "13": 31910.0, + "14": 33275.0, + "15": 30931.0, + "16": 31713.0, + "17": 31263.0, + "18": 30574.0, + "19": 31200.0, + "20": 28330.0, + "21": 28769.0, + "22": 27643.0, + "23": 34373.0, + "24": 28585.0, + "25": 27693.0, + "26": 30668.0, + "27": 31536.0, + "28": 33620.0, + "29": 34457.0, + "30": 29810.0, + "31": 27999.0, + "32": 33871.0, + "33": 35836.0, + "34": 30770.0, + "35": 32783.0, + "36": 33452.0, + "37": 36437.0, + "38": 34114.0, + "39": 37433.0, + "40": 35063.0, + "41": 34239.0, + "42": 36041.0, + "43": 33976.0, + "44": 33303.0, + "45": 35148.0, + "46": 36112.0, + "47": 38996.0, + "48": 34779.0, + "49": 34913.0, + "50": 38547.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1364012544.0, + "2": 1364020736.0, + "3": 1364009472.0, + "4": 1364020224.0, + "5": 1364007424.0, + "6": 1364005888.0, + "7": 1364011520.0, + "8": 1364006912.0, + "9": 1364019712.0, + "10": 1364001280.0, + "11": 1364009984.0, + "12": 1364022784.0, + "13": 1364008960.0, + "14": 1364003840.0, + "15": 1364007936.0, + "16": 1363994112.0, + "17": 1363993088.0, + "18": 1364017152.0, + "19": 1364009984.0, + "20": 1363999744.0, + "21": 1364006400.0, + "22": 1363983360.0, + "23": 1363997696.0, + "24": 1364001792.0, + "25": 1363992576.0, + "26": 1364003328.0, + "27": 1363998720.0, + "28": 1363982336.0, + "29": 1363995136.0, + "30": 1363979776.0, + "31": 1363991552.0, + "32": 1363983872.0, + "33": 1363983872.0, + "34": 1363997184.0, + "35": 1363991040.0, + "36": 1363984896.0, + "37": 1363998720.0, + "38": 1363991040.0, + "39": 1363976704.0, + "40": 1363971584.0, + "41": 1363983872.0, + "42": 1363977728.0, + "43": 1363967488.0, + "44": 1363974656.0, + "45": 1363974144.0, + "46": 1363965952.0, + "47": 1363952640.0, + "48": 1363951616.0, + "49": 1363960320.0, + "50": 1363956736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3404911104.0, + "2": 3972516352.0, + "3": 3976973312.0, + "4": 3976973312.0, + "5": 3976973312.0, + "6": 3976973312.0, + "7": 3976973312.0, + "8": 3976973312.0, + "9": 3976973312.0, + "10": 3976973312.0, + "11": 3976973312.0, + "12": 3976975872.0, + "13": 3976975872.0, + "14": 3976975872.0, + "15": 3976975872.0, + "16": 3976975872.0, + "17": 3976975872.0, + "18": 3976975872.0, + "19": 3976975872.0, + "20": 3976975872.0, + "21": 3976975872.0, + "22": 3976975872.0, + "23": 3976975872.0, + "24": 3976975872.0, + "25": 3976975872.0, + "26": 3976975872.0, + "27": 3976975872.0, + "28": 3976975872.0, + "29": 3976975872.0, + "30": 3976975872.0, + "31": 3976975872.0, + "32": 3976975872.0, + "33": 3976975872.0, + "34": 3976975872.0, + "35": 3976975872.0, + "36": 3976975872.0, + "37": 3976975872.0, + "38": 3976975872.0, + "39": 3976975872.0, + "40": 3976975872.0, + "41": 3976975872.0, + "42": 3976975872.0, + "43": 3976975872.0, + "44": 3976975872.0, + "45": 3976975872.0, + "46": 3976975872.0, + "47": 3976975872.0, + "48": 3976975872.0, + "49": 3976975872.0, + "50": 3976975872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.13471, + "2": 0.38969, + "3": 0.3103, + "4": 0.30799, + "5": 0.30603, + "6": 0.30643, + "7": 0.30599, + "8": 0.30393, + "9": 0.31467, + "10": 0.306, + "11": 0.30312, + "12": 0.30329, + "13": 0.30269, + "14": 0.30471, + "15": 0.30849, + "16": 0.30377, + "17": 0.30261, + "18": 0.3009, + "19": 0.30087, + "20": 0.30146, + "21": 0.30129, + "22": 0.30052, + "23": 0.30116, + "24": 0.2999, + "25": 0.30045, + "26": 0.30179, + "27": 0.30152, + "28": 0.30154, + "29": 0.30096, + "30": 0.31152, + "31": 0.31066, + "32": 0.30418, + "33": 0.35425, + "34": 0.30235, + "35": 0.30245, + "36": 0.302, + "37": 0.31083, + "38": 0.30275, + "39": 0.30205, + "40": 0.30289, + "41": 0.30375, + "42": 0.30405, + "43": 0.30174, + "44": 0.30207, + "45": 0.303, + "46": 0.30186, + "47": 0.30147, + "48": 0.30311, + "49": 0.30205, + "50": 0.3031 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json index e640ba5b102..9114b4bb385 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.8174, "5": 10.86097, "10": 10.80463, "15": 10.79539, "20": 10.71135, "25": 10.51962, "30": 10.33346, "35": 10.22683, "40": 10.041, "45": 9.7722, "50": 9.86932}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 26535.0, "5": 31546.0, "10": 25541.0, "15": 30463.0, "20": 28331.0, "25": 27263.0, "30": 29642.0, "35": 32022.0, "40": 34911.0, "45": 35017.0, "50": 38975.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1356172288.0, "5": 1356170240.0, "10": 1356160000.0, "15": 1356180480.0, "20": 1356170240.0, "25": 1356182528.0, "30": 1356198400.0, "35": 1356182016.0, "40": 1356156416.0, "45": 1356140544.0, "50": 1356098560.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3163797504.0, "5": 3751905792.0, "10": 3759570944.0, "15": 3759570944.0, "20": 3759570944.0, "25": 3759570944.0, "30": 3770054144.0, "35": 3770054144.0, "40": 3770054144.0, "45": 3770054144.0, "50": 3770054144.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 25.66526, "5": 0.29505, "10": 0.28776, "15": 0.28997, "20": 0.28516, "25": 0.29206, "30": 0.2932, "35": 0.29941, "40": 0.30125, "45": 0.29922, "50": 0.30133}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.78519, + "2": 10.81141, + "3": 10.79553, + "4": 10.78363, + "5": 10.83004, + "6": 10.8417, + "7": 10.81276, + "8": 10.81576, + "9": 10.82408, + "10": 10.75834, + "11": 10.86449, + "12": 10.83694, + "13": 10.84627, + "14": 10.85637, + "15": 10.79099, + "16": 10.78206, + "17": 10.73911, + "18": 10.77969, + "19": 10.77703, + "20": 10.69044, + "21": 10.66086, + "22": 10.51537, + "23": 10.68288, + "24": 10.54469, + "25": 10.49929, + "26": 10.5582, + "27": 10.57541, + "28": 10.54334, + "29": 10.55412, + "30": 10.31857, + "31": 10.06911, + "32": 10.42007, + "33": 10.42435, + "34": 10.16897, + "35": 10.22944, + "36": 10.19358, + "37": 10.29976, + "38": 10.1465, + "39": 10.37282, + "40": 10.04287, + "41": 10.09714, + "42": 10.17339, + "43": 9.79317, + "44": 9.91871, + "45": 9.79389, + "46": 9.77586, + "47": 10.10721, + "48": 9.81704, + "49": 9.48887, + "50": 9.87988 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 26764.0, + "2": 29274.0, + "3": 28827.0, + "4": 27434.0, + "5": 31796.0, + "6": 32589.0, + "7": 31434.0, + "8": 27373.0, + "9": 30359.0, + "10": 25775.0, + "11": 33420.0, + "12": 30835.0, + "13": 31910.0, + "14": 33275.0, + "15": 30931.0, + "16": 31713.0, + "17": 31263.0, + "18": 30574.0, + "19": 31200.0, + "20": 28330.0, + "21": 28769.0, + "22": 27643.0, + "23": 34373.0, + "24": 28585.0, + "25": 27693.0, + "26": 30668.0, + "27": 31536.0, + "28": 33620.0, + "29": 34457.0, + "30": 29810.0, + "31": 27999.0, + "32": 33871.0, + "33": 35836.0, + "34": 30770.0, + "35": 32783.0, + "36": 33452.0, + "37": 36437.0, + "38": 34114.0, + "39": 37433.0, + "40": 35063.0, + "41": 34239.0, + "42": 36041.0, + "43": 33976.0, + "44": 33303.0, + "45": 35148.0, + "46": 36112.0, + "47": 38996.0, + "48": 34779.0, + "49": 34913.0, + "50": 38547.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1364012544.0, + "2": 1364020736.0, + "3": 1364009472.0, + "4": 1364020224.0, + "5": 1364007424.0, + "6": 1364005888.0, + "7": 1364011520.0, + "8": 1364006912.0, + "9": 1364019712.0, + "10": 1364001280.0, + "11": 1364009984.0, + "12": 1364022784.0, + "13": 1364008960.0, + "14": 1364003840.0, + "15": 1364007936.0, + "16": 1363994112.0, + "17": 1363993088.0, + "18": 1364017152.0, + "19": 1364009984.0, + "20": 1363999744.0, + "21": 1364006400.0, + "22": 1363983360.0, + "23": 1363997696.0, + "24": 1364001792.0, + "25": 1363992576.0, + "26": 1364003328.0, + "27": 1363998720.0, + "28": 1363982336.0, + "29": 1363995136.0, + "30": 1363979776.0, + "31": 1363991552.0, + "32": 1363983872.0, + "33": 1363983872.0, + "34": 1363997184.0, + "35": 1363991040.0, + "36": 1363984896.0, + "37": 1363998720.0, + "38": 1363991040.0, + "39": 1363976704.0, + "40": 1363971584.0, + "41": 1363983872.0, + "42": 1363977728.0, + "43": 1363967488.0, + "44": 1363974656.0, + "45": 1363974144.0, + "46": 1363965952.0, + "47": 1363952640.0, + "48": 1363951616.0, + "49": 1363960320.0, + "50": 1363956736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3404911104.0, + "2": 3972516352.0, + "3": 3976973312.0, + "4": 3976973312.0, + "5": 3976973312.0, + "6": 3976973312.0, + "7": 3976973312.0, + "8": 3976973312.0, + "9": 3976973312.0, + "10": 3976973312.0, + "11": 3976973312.0, + "12": 3976975872.0, + "13": 3976975872.0, + "14": 3976975872.0, + "15": 3976975872.0, + "16": 3976975872.0, + "17": 3976975872.0, + "18": 3976975872.0, + "19": 3976975872.0, + "20": 3976975872.0, + "21": 3976975872.0, + "22": 3976975872.0, + "23": 3976975872.0, + "24": 3976975872.0, + "25": 3976975872.0, + "26": 3976975872.0, + "27": 3976975872.0, + "28": 3976975872.0, + "29": 3976975872.0, + "30": 3976975872.0, + "31": 3976975872.0, + "32": 3976975872.0, + "33": 3976975872.0, + "34": 3976975872.0, + "35": 3976975872.0, + "36": 3976975872.0, + "37": 3976975872.0, + "38": 3976975872.0, + "39": 3976975872.0, + "40": 3976975872.0, + "41": 3976975872.0, + "42": 3976975872.0, + "43": 3976975872.0, + "44": 3976975872.0, + "45": 3976975872.0, + "46": 3976975872.0, + "47": 3976975872.0, + "48": 3976975872.0, + "49": 3976975872.0, + "50": 3976975872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.62035, + "2": 0.36752, + "3": 0.30562, + "4": 0.29876, + "5": 0.298, + "6": 0.29743, + "7": 0.29729, + "8": 0.2967, + "9": 0.29751, + "10": 0.29912, + "11": 0.29575, + "12": 0.29589, + "13": 0.29696, + "14": 0.29898, + "15": 0.30053, + "16": 0.30093, + "17": 0.2977, + "18": 0.2973, + "19": 0.29596, + "20": 0.29757, + "21": 0.2967, + "22": 0.29963, + "23": 0.29707, + "24": 0.29748, + "25": 0.29701, + "26": 0.29838, + "27": 0.29889, + "28": 0.29962, + "29": 0.30399, + "30": 0.30932, + "31": 0.30553, + "32": 0.29765, + "33": 0.30499, + "34": 0.29754, + "35": 0.29747, + "36": 0.29801, + "37": 0.30768, + "38": 0.29693, + "39": 0.29912, + "40": 0.299, + "41": 0.2982, + "42": 0.37256, + "43": 0.29865, + "44": 0.29774, + "45": 0.29961, + "46": 0.2988, + "47": 0.30454, + "48": 0.30466, + "49": 0.30093, + "50": 0.29883 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json new file mode 100644 index 00000000000..390f4b8f13f --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.77518, + "2": 10.78038, + "3": 10.79302, + "4": 10.74107, + "5": 10.82013, + "6": 10.82951, + "7": 10.7953, + "8": 10.78263, + "9": 10.79278, + "10": 10.7446, + "11": 10.85147, + "12": 10.82613, + "13": 10.82825, + "14": 10.85504, + "15": 10.75536, + "16": 10.75777, + "17": 10.72319, + "18": 10.76274, + "19": 10.75075, + "20": 10.66587, + "21": 10.6419, + "22": 10.47523, + "23": 10.66959, + "24": 10.54157, + "25": 10.4825, + "26": 10.55255, + "27": 10.57459, + "28": 10.55159, + "29": 10.5668, + "30": 10.31134, + "31": 10.01921, + "32": 10.42655, + "33": 10.42294, + "34": 10.14739, + "35": 10.21574, + "36": 10.15811, + "37": 10.30279, + "38": 10.14031, + "39": 10.36301, + "40": 10.02669, + "41": 10.07635, + "42": 10.16156, + "43": 9.74374, + "44": 9.88962, + "45": 9.75874, + "46": 9.73618, + "47": 10.0844, + "48": 9.78532, + "49": 9.45072, + "50": 9.85634 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 27105.0, + "2": 28791.0, + "3": 29282.0, + "4": 27583.0, + "5": 31595.0, + "6": 32831.0, + "7": 31023.0, + "8": 27107.0, + "9": 30780.0, + "10": 25505.0, + "11": 33684.0, + "12": 30235.0, + "13": 32960.0, + "14": 32880.0, + "15": 30405.0, + "16": 32455.0, + "17": 30933.0, + "18": 30623.0, + "19": 30803.0, + "20": 28593.0, + "21": 29002.0, + "22": 27030.0, + "23": 34463.0, + "24": 29154.0, + "25": 27827.0, + "26": 31119.0, + "27": 32108.0, + "28": 33412.0, + "29": 34737.0, + "30": 30465.0, + "31": 28775.0, + "32": 33115.0, + "33": 34745.0, + "34": 30785.0, + "35": 32116.0, + "36": 33968.0, + "37": 36757.0, + "38": 34150.0, + "39": 37240.0, + "40": 35353.0, + "41": 34638.0, + "42": 36703.0, + "43": 34601.0, + "44": 33783.0, + "45": 35388.0, + "46": 35484.0, + "47": 40591.0, + "48": 36671.0, + "49": 36174.0, + "50": 38231.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1566715392.0, + "2": 1564834816.0, + "3": 1564860416.0, + "4": 1564313088.0, + "5": 1564120576.0, + "6": 1565958144.0, + "7": 1564614656.0, + "8": 1563520512.0, + "9": 1563819008.0, + "10": 1563891712.0, + "11": 1563430400.0, + "12": 1564003840.0, + "13": 1564584960.0, + "14": 1566458880.0, + "15": 1564517376.0, + "16": 1565116928.0, + "17": 1564445184.0, + "18": 1566232576.0, + "19": 1565403136.0, + "20": 1565009408.0, + "21": 1568513024.0, + "22": 1564807168.0, + "23": 1567177216.0, + "24": 1564333568.0, + "25": 1566986240.0, + "26": 1565763584.0, + "27": 1563955200.0, + "28": 1564926464.0, + "29": 1563690496.0, + "30": 1563930624.0, + "31": 1564862464.0, + "32": 1564173824.0, + "33": 1565158400.0, + "34": 1564895232.0, + "35": 1564069376.0, + "36": 1565074944.0, + "37": 1563990016.0, + "38": 1565044736.0, + "39": 1563646976.0, + "40": 1565059072.0, + "41": 1566659584.0, + "42": 1564623360.0, + "43": 1563140608.0, + "44": 1565100032.0, + "45": 1567626752.0, + "46": 1565074944.0, + "47": 1564590080.0, + "48": 1564522496.0, + "49": 1563403776.0, + "50": 1564356096.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3737165824.0, + "2": 4315653632.0, + "3": 4318730752.0, + "4": 4318730752.0, + "5": 4318730752.0, + "6": 4346984960.0, + "7": 4346984960.0, + "8": 4346984960.0, + "9": 4346984960.0, + "10": 4346984960.0, + "11": 4346984960.0, + "12": 4346984960.0, + "13": 4346984960.0, + "14": 4346984960.0, + "15": 4346984960.0, + "16": 4346984960.0, + "17": 4346984960.0, + "18": 4346984960.0, + "19": 4346984960.0, + "20": 4346984960.0, + "21": 4346984960.0, + "22": 4346984960.0, + "23": 4346984960.0, + "24": 4346984960.0, + "25": 4346984960.0, + "26": 4346984960.0, + "27": 4346984960.0, + "28": 4346984960.0, + "29": 4346984960.0, + "30": 4346984960.0, + "31": 4346984960.0, + "32": 4346984960.0, + "33": 4346984960.0, + "34": 4346984960.0, + "35": 4346984960.0, + "36": 4346984960.0, + "37": 4346984960.0, + "38": 4346984960.0, + "39": 4346984960.0, + "40": 4346984960.0, + "41": 4346984960.0, + "42": 4346984960.0, + "43": 4346984960.0, + "44": 4346984960.0, + "45": 4346984960.0, + "46": 4346984960.0, + "47": 4346984960.0, + "48": 4346984960.0, + "49": 4346984960.0, + "50": 4346984960.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 19.53588, + "2": 0.42103, + "3": 0.37171, + "4": 0.32787, + "5": 0.32504, + "6": 0.29089, + "7": 0.29215, + "8": 0.28165, + "9": 0.287, + "10": 0.28805, + "11": 0.27222, + "12": 0.2768, + "13": 0.26942, + "14": 0.27369, + "15": 0.27287, + "16": 0.26492, + "17": 0.27231, + "18": 0.26408, + "19": 0.26251, + "20": 0.26769, + "21": 0.26539, + "22": 0.27591, + "23": 0.26191, + "24": 0.27512, + "25": 0.26855, + "26": 0.27323, + "27": 0.26526, + "28": 0.26748, + "29": 0.27054, + "30": 0.28834, + "31": 0.27784, + "32": 0.27006, + "33": 0.29268, + "34": 0.27918, + "35": 0.29002, + "36": 0.28672, + "37": 0.28354, + "38": 0.27713, + "39": 0.28145, + "40": 0.62986, + "41": 0.2715, + "42": 0.2633, + "43": 0.27508, + "44": 0.28176, + "45": 0.26745, + "46": 0.27357, + "47": 0.26873, + "48": 0.26907, + "49": 0.26843, + "50": 0.63541 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..779e572fdd4 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.59918, + "2": 0.61895, + "3": 0.56068, + "4": 0.55629, + "5": 0.55614, + "6": 0.55955, + "7": 0.55763, + "8": 0.55984, + "9": 0.55, + "10": 0.5499, + "11": 0.55012, + "12": 0.54844, + "13": 0.55795, + "14": 0.54851, + "15": 0.5504, + "16": 0.55068, + "17": 0.54716, + "18": 0.55003, + "19": 0.54872, + "20": 0.54841, + "21": 0.55411, + "22": 0.55114, + "23": 0.54838, + "24": 0.54877, + "25": 0.54811, + "26": 0.54808, + "27": 0.54959, + "28": 0.55486, + "29": 0.55193, + "30": 0.54966, + "31": 0.54964, + "32": 0.55053, + "33": 0.55025, + "34": 0.55174, + "35": 0.55, + "36": 0.55117, + "37": 0.55004, + "38": 0.55089, + "39": 0.54857, + "40": 0.55402, + "41": 0.55168, + "42": 0.54916, + "43": 0.55132, + "44": 0.54941, + "45": 0.54876, + "46": 0.55724, + "47": 0.54849, + "48": 0.55007, + "49": 0.55215, + "50": 0.54754 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json index 7aae0170419..a186febffbe 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.79288, "5": 10.828, "10": 10.76831, "15": 10.79073, "20": 10.7066, "25": 10.54061, "30": 10.39485, "35": 10.30488, "40": 10.13079, "45": 9.90236, "50": 9.97938}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 5601.0, "5": 6601.0, "10": 5254.0, "15": 6250.0, "20": 5969.0, "25": 5816.0, "30": 6181.0, "35": 6751.0, "40": 6831.0, "45": 6881.0, "50": 7684.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 458211840.0, "5": 458212864.0, "10": 458214912.0, "15": 458215424.0, "20": 458212864.0, "25": 458215424.0, "30": 458211840.0, "35": 458214400.0, "40": 458214912.0, "45": 458213888.0, "50": 458214912.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1016394240.0, "5": 1180904960.0, "10": 1180934144.0, "15": 1180990976.0, "20": 1180990976.0, "25": 1181222912.0, "30": 1181222912.0, "35": 1181468160.0, "40": 1181468160.0, "45": 1183467008.0, "50": 1183467008.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 16.31521, "5": 0.56321, "10": 0.56536, "15": 0.56469, "20": 0.55362, "25": 0.55331, "30": 0.55482, "35": 0.57032, "40": 0.56489, "45": 0.55226, "50": 0.54242}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.49228, + "2": 0.63481, + "3": 0.56951, + "4": 0.57807, + "5": 0.581, + "6": 0.58159, + "7": 0.5705, + "8": 0.56929, + "9": 0.56794, + "10": 0.56314, + "11": 0.57935, + "12": 0.57294, + "13": 0.56865, + "14": 0.56698, + "15": 0.56505, + "16": 0.56266, + "17": 0.56337, + "18": 0.56237, + "19": 0.56197, + "20": 0.5626, + "21": 0.5642, + "22": 0.56373, + "23": 0.57291, + "24": 0.56432, + "25": 0.56287, + "26": 0.56295, + "27": 0.56146, + "28": 0.56459, + "29": 0.56415, + "30": 0.56587, + "31": 0.5671, + "32": 0.56896, + "33": 0.57526, + "34": 0.57281, + "35": 0.57407, + "36": 0.57321, + "37": 0.57403, + "38": 0.57296, + "39": 0.57248, + "40": 0.57089, + "41": 0.57201, + "42": 0.5661, + "43": 0.57044, + "44": 0.56777, + "45": 0.56877, + "46": 0.57143, + "47": 0.57031, + "48": 0.56952, + "49": 0.57353, + "50": 0.56636 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json new file mode 100644 index 00000000000..63611a809be --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.94206, + "2": 0.63285, + "3": 0.57182, + "4": 0.57773, + "5": 0.57872, + "6": 0.57643, + "7": 0.57659, + "8": 0.57355, + "9": 0.57629, + "10": 0.56867, + "11": 0.56833, + "12": 0.56771, + "13": 0.56904, + "14": 0.59894, + "15": 0.67038, + "16": 0.5641, + "17": 0.56097, + "18": 0.56495, + "19": 0.56059, + "20": 0.56119, + "21": 0.55935, + "22": 0.55364, + "23": 0.55179, + "24": 0.55091, + "25": 0.55193, + "26": 0.55148, + "27": 0.55193, + "28": 0.55181, + "29": 0.55146, + "30": 0.55214, + "31": 0.55292, + "32": 0.55164, + "33": 0.55141, + "34": 0.54974, + "35": 0.55215, + "36": 0.55139, + "37": 0.55509, + "38": 0.5519, + "39": 0.55134, + "40": 0.55133, + "41": 0.55143, + "42": 0.5495, + "43": 0.54991, + "44": 0.5491, + "45": 0.54833, + "46": 0.54896, + "47": 0.54857, + "48": 0.5489, + "49": 0.54919, + "50": 0.54936 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json index 5f0c72de4dd..4ebfff8da76 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.79288, "5": 10.828, "10": 10.76831, "15": 10.79073, "20": 10.7066, "25": 10.54061, "30": 10.39485, "35": 10.30488, "40": 10.13079, "45": 9.90236, "50": 9.97938}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 5601.0, "5": 6601.0, "10": 5254.0, "15": 6250.0, "20": 5969.0, "25": 5816.0, "30": 6181.0, "35": 6751.0, "40": 6831.0, "45": 6881.0, "50": 7684.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 458211840.0, "5": 458212864.0, "10": 458214912.0, "15": 458215424.0, "20": 458212864.0, "25": 458215424.0, "30": 458211840.0, "35": 458214400.0, "40": 458214912.0, "45": 458213888.0, "50": 458214912.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1016394240.0, "5": 1180904960.0, "10": 1180934144.0, "15": 1180990976.0, "20": 1180990976.0, "25": 1181222912.0, "30": 1181222912.0, "35": 1181468160.0, "40": 1181468160.0, "45": 1183467008.0, "50": 1183467008.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 19.9711, "5": 0.5845, "10": 0.59728, "15": 0.60305, "20": 0.58013, "25": 0.57491, "30": 0.5625, "35": 0.56368, "40": 0.56421, "45": 0.57011, "50": 0.58956}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.27777, + "2": 0.60806, + "3": 0.55409, + "4": 0.55324, + "5": 0.54815, + "6": 0.54698, + "7": 0.54712, + "8": 0.55008, + "9": 0.55718, + "10": 0.55527, + "11": 0.55082, + "12": 0.56208, + "13": 0.55625, + "14": 0.55717, + "15": 0.56582, + "16": 0.55953, + "17": 0.57188, + "18": 0.55508, + "19": 0.55956, + "20": 0.55934, + "21": 0.55676, + "22": 0.55842, + "23": 0.55867, + "24": 0.55987, + "25": 0.55941, + "26": 0.55642, + "27": 0.55364, + "28": 0.55209, + "29": 0.55397, + "30": 0.55602, + "31": 0.55344, + "32": 0.55195, + "33": 0.56308, + "34": 0.55588, + "35": 0.55251, + "36": 0.55314, + "37": 0.55563, + "38": 0.56708, + "39": 0.5661, + "40": 0.56725, + "41": 0.5663, + "42": 0.56565, + "43": 0.5725, + "44": 0.56736, + "45": 0.5674, + "46": 0.56751, + "47": 0.56642, + "48": 0.56257, + "49": 0.56841, + "50": 0.56452 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json new file mode 100644 index 00000000000..2c1e24efd92 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.85676, + "2": 0.38679, + "3": 0.33955, + "4": 0.34149, + "5": 0.33738, + "6": 0.3367, + "7": 0.33549, + "8": 0.33607, + "9": 0.33582, + "10": 0.33626, + "11": 0.33555, + "12": 0.33476, + "13": 0.33518, + "14": 0.33155, + "15": 0.33236, + "16": 0.33276, + "17": 0.33237, + "18": 0.33234, + "19": 0.33191, + "20": 0.33586, + "21": 0.334, + "22": 0.33303, + "23": 0.33299, + "24": 0.33339, + "25": 0.33301, + "26": 0.33368, + "27": 0.33213, + "28": 0.33242, + "29": 0.33251, + "30": 0.33285, + "31": 0.33233, + "32": 0.33175, + "33": 0.33133, + "34": 0.33224, + "35": 0.3317, + "36": 0.33247, + "37": 0.33189, + "38": 0.3324, + "39": 0.33157, + "40": 0.33224, + "41": 0.33218, + "42": 0.33335, + "43": 0.33381, + "44": 0.33284, + "45": 0.33399, + "46": 0.33341, + "47": 0.33248, + "48": 0.33301, + "49": 0.33248, + "50": 0.33145 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..c1d418bd4a8 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.62289, + "2": 0.39639, + "3": 0.35696, + "4": 0.3674, + "5": 0.3577, + "6": 0.35509, + "7": 0.35808, + "8": 0.35462, + "9": 0.35492, + "10": 0.35391, + "11": 0.35546, + "12": 0.35515, + "13": 0.35432, + "14": 0.35491, + "15": 0.35335, + "16": 0.35396, + "17": 0.35304, + "18": 0.35201, + "19": 0.35198, + "20": 0.35194, + "21": 0.35412, + "22": 0.35208, + "23": 0.3541, + "24": 0.3519, + "25": 0.38526, + "26": 0.39355, + "27": 0.39377, + "28": 0.3934, + "29": 0.39406, + "30": 0.39393, + "31": 0.38905, + "32": 0.3893, + "33": 0.39052, + "34": 0.39004, + "35": 0.38977, + "36": 0.39027, + "37": 0.38961, + "38": 0.38929, + "39": 0.38944, + "40": 0.38968, + "41": 0.38938, + "42": 0.38964, + "43": 0.3909, + "44": 0.39131, + "45": 0.38436, + "46": 0.34892, + "47": 0.34845, + "48": 0.34853, + "49": 0.3474, + "50": 0.34681 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json index 1f335e82393..478bae6fdec 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82416, "5": 10.84149, "10": 10.79041, "15": 10.8302, "20": 10.73076, "25": 10.57969, "30": 10.41029, "35": 10.31774, "40": 10.14481, "45": 9.91307, "50": 9.97569}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 12709.0, "5": 15699.0, "10": 12503.0, "15": 14756.0, "20": 13658.0, "25": 13295.0, "30": 14801.0, "35": 15233.0, "40": 16737.0, "45": 16130.0, "50": 19242.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 625788416.0, "5": 625793536.0, "10": 625788928.0, "15": 625790976.0, "20": 625793024.0, "25": 625789440.0, "30": 625790976.0, "35": 625789952.0, "40": 625793536.0, "45": 625790976.0, "50": 625793024.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1818507264.0, "5": 2049478144.0, "10": 2049478144.0, "15": 2050050048.0, "20": 2050050048.0, "25": 2050050048.0, "30": 2050050048.0, "35": 2050050048.0, "40": 2050050048.0, "45": 2050050048.0, "50": 2050050048.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 16.64702, "5": 0.34446, "10": 0.34582, "15": 0.34645, "20": 0.34853, "25": 0.34852, "30": 0.34703, "35": 0.34516, "40": 0.34201, "45": 0.34861, "50": 0.3491}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.46534, + "2": 0.4102, + "3": 0.34219, + "4": 0.34093, + "5": 0.34255, + "6": 0.33706, + "7": 0.33661, + "8": 0.33616, + "9": 0.33565, + "10": 0.33568, + "11": 0.33538, + "12": 0.33457, + "13": 0.33488, + "14": 0.33416, + "15": 0.33683, + "16": 0.33684, + "17": 0.33708, + "18": 0.33815, + "19": 0.33805, + "20": 0.33696, + "21": 0.33675, + "22": 0.33623, + "23": 0.33752, + "24": 0.33699, + "25": 0.3409, + "26": 0.33513, + "27": 0.33524, + "28": 0.33491, + "29": 0.33714, + "30": 0.33571, + "31": 0.33638, + "32": 0.33629, + "33": 0.3369, + "34": 0.33685, + "35": 0.33651, + "36": 0.33539, + "37": 0.33561, + "38": 0.33636, + "39": 0.33558, + "40": 0.3356, + "41": 0.33618, + "42": 0.33669, + "43": 0.33535, + "44": 0.3362, + "45": 0.3354, + "46": 0.33686, + "47": 0.33486, + "48": 0.33657, + "49": 0.33563, + "50": 0.33513 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json index 51d434d10cf..1a9705f8181 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json @@ -174,5 +174,5 @@ -0.5394397377967834 ] }, - "throughput": 25.35687538450034 + "throughput": [25.35687538450034, 25.35687538450034] } diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json index 711eeddfb25..7370d3c6bb0 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/golden_values_dev_dgx_h100.json @@ -157,5 +157,5 @@ -0.5394397377967834 ] }, - "throughput": 6.543502517233578 + "throughput": [6.543502517233578, 6.543502517233578] } \ No newline at end of file diff --git a/tests/test_utils/python_scripts/download_golden_values.py b/tests/test_utils/python_scripts/download_golden_values.py index 650867f231f..e2294b32fbb 100644 --- a/tests/test_utils/python_scripts/download_golden_values.py +++ b/tests/test_utils/python_scripts/download_golden_values.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import logging import os import pathlib @@ -91,8 +93,6 @@ def main(pipeline_id: int, only_failing: bool): continue golden_values_source_name = golden_values_source.name - golden_values_source_name = golden_values_source_name.replace("_dgx_h100", "") - golden_values_source_name = golden_values_source_name.replace("_dgx_a100", "") golden_values_source_name = golden_values_source_name.replace( "generations", "golden_values" ) diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py index 7dc4a7357a7..846ca8a1521 100644 --- a/tests/test_utils/python_scripts/launch_jet_workload.py +++ b/tests/test_utils/python_scripts/launch_jet_workload.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import json import logging import os @@ -289,6 +291,7 @@ def is_flaky_failure(concat_allranks_logs: str) -> bool: or "unspecified launch failure" in concat_allranks_logs or "free(): corrupted unsorted chunks" in concat_allranks_logs or "Segfault encountered" in concat_allranks_logs + or "Fatal glibc error" in concat_allranks_logs ) diff --git a/tests/unit_tests/inference/contexts/test_dynamic_context.py b/tests/unit_tests/inference/contexts/test_dynamic_context.py index 1cd9d66ece1..0674cdfcabd 100644 --- a/tests/unit_tests/inference/contexts/test_dynamic_context.py +++ b/tests/unit_tests/inference/contexts/test_dynamic_context.py @@ -12,6 +12,7 @@ ) from megatron.core.inference.inference_request import DynamicInferenceRequest from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.ssm.mamba_hybrid_layer_allocation import Symbols from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from tests.unit_tests.test_utilities import Utils @@ -42,12 +43,19 @@ def _get_dynamic_context( max_sequence_length, buffer_size_gb, block_size_tokens, - buffer_guarenteed_fraction, + buffer_guaranteed_fraction, buffer_overflow_factor, max_requests_override, max_tokens_override, + is_hybrid_model=False, + layer_type_list=None, + rounder=64, ): - set_rounder(64) + set_rounder(rounder) + + if is_hybrid_model and layer_type_list is None: + layer_type_list = [Symbols.MAMBA, Symbols.MLP, Symbols.ATTENTION, Symbols.MLP] + dynamic_context = DynamicInferenceContext( params_dtype=params_dtype, num_layers=num_layers, @@ -55,23 +63,27 @@ def _get_dynamic_context( num_attention_heads=num_attention_heads, max_sequence_length=max_sequence_length, num_cuda_graphs=None, + use_cuda_graphs_for_non_decode_steps=not is_hybrid_model, buffer_size_gb=buffer_size_gb, - buffer_guaranteed_fraction=buffer_guarenteed_fraction, + buffer_guaranteed_fraction=buffer_guaranteed_fraction, block_size_tokens=block_size_tokens, buffer_overflow_factor=buffer_overflow_factor, max_requests_override=max_requests_override, max_tokens_override=max_tokens_override, + layer_type_list=layer_type_list, + mamba_conv_states_shape=(544, 4), + mamba_ssm_states_shape=(8, 64, 16), use_flashinfer_fused_rope=None, # default to using flash-infer if available # this is for compatibility with the LTS environment ) return dynamic_context def teardown_method(self, method): - set_rounder(64) Utils.destroy_model_parallel() @pytest.mark.internal - def test_initialize_dynamic_context(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_initialize_dynamic_context(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) dynamic_context = self._get_dynamic_context( @@ -81,18 +93,30 @@ def test_initialize_dynamic_context(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) - assert dynamic_context.gtd_block_count == 48 - assert dynamic_context.gtd_request_count == 12 - assert dynamic_context.block_allocator.block_count_total == 491 - assert dynamic_context.max_requests == 128 - assert dynamic_context.max_tokens == 62848 + if not is_hybrid_model: + assert dynamic_context.gtd_block_count == 48 + assert dynamic_context.gtd_request_count == 12 + assert dynamic_context.block_allocator.block_count_total == 491 + assert dynamic_context.max_requests == 128 + assert dynamic_context.max_tokens == 62848 + assert dynamic_context.num_mamba_layers == 0 + assert dynamic_context.mamba_metadata is None + else: + assert dynamic_context.gtd_block_count == 112 + assert dynamic_context.gtd_request_count == 28 + assert dynamic_context.block_allocator.block_count_total == 1156 + assert dynamic_context.max_requests == 320 + assert dynamic_context.max_tokens == 154176 + assert dynamic_context.num_mamba_layers == 1 + assert dynamic_context.mamba_metadata is not None # Check initializations to -1 assert torch.all(dynamic_context.request_ids == -1) @@ -100,32 +124,38 @@ def test_initialize_dynamic_context(self): @pytest.mark.internal def test_is_static_batching(self): self._setup_model_parallel_group(1, 1) - dynamic_context = DynamicInferenceContext( + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=2, kv_channels=64, num_attention_heads=8, max_sequence_length=512, - num_cuda_graphs=None, buffer_size_gb=1.0, buffer_guaranteed_fraction=0.1, block_size_tokens=128, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, ) assert not dynamic_context.is_static_batching() @pytest.mark.internal - def test_is_memory_available(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_is_memory_available(self, is_hybrid_model): self._setup_model_parallel_group(1, 1) - dynamic_context = DynamicInferenceContext( + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=2, kv_channels=64, num_attention_heads=8, max_sequence_length=512, - num_cuda_graphs=None, buffer_size_gb=1.0, buffer_guaranteed_fraction=0.1, block_size_tokens=128, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) dynamic_context.block_allocator.block_count_avail = 10 assert dynamic_context.block_allocator.is_memory_available(10) @@ -141,19 +171,24 @@ def test_is_memory_available(self): assert not dynamic_context.block_allocator.is_memory_available(6, safe=True) @pytest.mark.internal - def test_request_overflow(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_request_overflow(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) - set_rounder(1) - dynamic_context = DynamicInferenceContext( + + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=2, kv_channels=64, num_attention_heads=8, max_sequence_length=128, - num_cuda_graphs=None, buffer_size_gb=0.01, buffer_guaranteed_fraction=0.1, block_size_tokens=32, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, + rounder=1, + is_hybrid_model=is_hybrid_model, ) with pytest.raises(RequestOverflowError): for i in range(dynamic_context.max_requests + 1): @@ -168,22 +203,24 @@ def test_request_overflow(self): ) # Adding more than allowed requests @pytest.mark.internal - def test_token_overflow_error(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_token_overflow_error(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) - set_rounder(1) - dynamic_context = DynamicInferenceContext( + + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=2, kv_channels=64, num_attention_heads=8, max_sequence_length=512, - num_cuda_graphs=None, buffer_size_gb=0.1, buffer_guaranteed_fraction=0.1, block_size_tokens=128, buffer_overflow_factor=1.0, max_requests_override=2, max_tokens_override=20, # Setting a very low token limit + rounder=1, + is_hybrid_model=is_hybrid_model, ) with pytest.raises(TokenOverflowError): @@ -198,18 +235,23 @@ def test_token_overflow_error(self): ) # Exceeding max token count @pytest.mark.internal - def test_reset(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_reset(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) - dynamic_context = DynamicInferenceContext( + + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=2, kv_channels=64, num_attention_heads=8, max_sequence_length=128, - num_cuda_graphs=None, buffer_size_gb=1.0, buffer_guaranteed_fraction=0.1, block_size_tokens=128, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) # Initialize all variables @@ -234,6 +276,9 @@ def test_reset(self): dynamic_context.block_allocator.block_count_avail = 5 dynamic_context.memory_buffer.fill_(1) dynamic_context.request_to_kv_block_ids.fill_(1) + if is_hybrid_model: + dynamic_context.mamba_conv_states.fill_(1) + dynamic_context.mamba_ssm_states.fill_(1) # Call reset dynamic_context.reset() @@ -262,9 +307,14 @@ def test_reset(self): == dynamic_context.block_allocator.block_count_total - 1 ) assert torch.all(dynamic_context.request_to_kv_block_ids == -1) + if is_hybrid_model: + assert torch.all(dynamic_context.mamba_metadata.request_to_mamba_state_idx == -1) + assert torch.all(dynamic_context.mamba_conv_states == 0) + assert torch.all(dynamic_context.mamba_ssm_states == 0) @pytest.mark.internal - def test_allocate_and_release_memory_blocks(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_allocate_and_release_memory_blocks(self, is_hybrid_model): self._setup_model_parallel_group(1, 1) dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, @@ -273,23 +323,38 @@ def test_allocate_and_release_memory_blocks(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) - assert dynamic_context.block_allocator.allocate_memory_blocks( - 4 - ).cpu().detach().numpy().tolist() == [486, 487, 488, 489] - assert dynamic_context.block_allocator.block_count_avail == 486 + if is_hybrid_model: + expected_memory_blocks = [1151, 1152, 1153, 1154] + else: + expected_memory_blocks = [486, 487, 488, 489] + expected_block_count_avail = expected_memory_blocks[0] + + assert ( + dynamic_context.block_allocator.allocate_memory_blocks(4) + .cpu() + .detach() + .numpy() + .tolist() + == expected_memory_blocks + ) + assert dynamic_context.block_allocator.block_count_avail == expected_block_count_avail dynamic_context.block_allocator.release_memory_blocks( - torch.tensor([488, 489], device='cuda') + torch.tensor(expected_memory_blocks[-2:], device='cuda') ) - assert dynamic_context.block_allocator.block_count_avail == 488 - assert dynamic_context.block_allocator.allocate_memory_blocks(1).item() == 489 - assert dynamic_context.block_allocator.block_count_avail == 487 + assert dynamic_context.block_allocator.block_count_avail == expected_block_count_avail + 2 + assert ( + dynamic_context.block_allocator.allocate_memory_blocks(1).item() + == expected_memory_blocks[-1] + ) + assert dynamic_context.block_allocator.block_count_avail == expected_block_count_avail + 1 # Should return None since we allocate more blocks than what we have. assert ( dynamic_context.block_allocator.allocate_memory_blocks( @@ -299,8 +364,10 @@ def test_allocate_and_release_memory_blocks(self): ) @pytest.mark.internal - def test_add_request(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_add_request(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=4, @@ -308,11 +375,12 @@ def test_add_request(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) assert dynamic_context.block_size_tokens == 128 context_length = 144 @@ -331,14 +399,10 @@ def test_add_request(self): assert torch.all(dynamic_context.request_ids[1:] == -1) assert dynamic_context.request_query_lengths[0] == context_length assert dynamic_context.request_kv_length_offsets[0] == 0 - assert dynamic_context.request_to_kv_block_ids[0].cpu().detach().numpy().tolist() == [ - 488, - 489, - -1, - -1, - ] assert dynamic_context.request_kv_block_counts[0] == 2 - assert dynamic_context.request_last_kv_block_id[0] == 489 + assert dynamic_context.request_last_kv_block_id[0].item() == ( + 1154 if is_hybrid_model else 489 + ) assert dynamic_context.request_last_kv_block_offset[0].item() == 15 assert torch.all( dynamic_context.token_to_pos_ids[0:context_length] @@ -352,17 +416,22 @@ def test_add_request(self): dynamic_context.token_to_position_in_request[0:context_length] == torch.arange(0, context_length, dtype=torch.long, device='cuda') ) + + # Verify token_to_block_idx and token_to_local_position_within_kv_block based on assigned blocks + first_block_id = dynamic_context.request_to_kv_block_ids[0, 0] + second_block_id = dynamic_context.request_to_kv_block_ids[0, 1] + assert torch.all( dynamic_context.token_to_block_idx[0:context_length][ 0 : dynamic_context.block_size_tokens ] - == 488 + == first_block_id ) assert torch.all( dynamic_context.token_to_block_idx[0:context_length][ dynamic_context.block_size_tokens : context_length ] - == 489 + == second_block_id ) assert torch.all( dynamic_context.token_to_local_position_within_kv_block[0:context_length] @@ -371,8 +440,10 @@ def test_add_request(self): ) @pytest.mark.internal - def test_update_request(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_update_request(self, is_hybrid_model: bool): self._setup_model_parallel_group(1, 1) + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=4, @@ -380,11 +451,12 @@ def test_update_request(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) # This case should just reset and return since all requests are finished @@ -394,10 +466,19 @@ def test_update_request(self): dynamic_context.request_kv_block_counts[0:3] = 1 new_block_ids = dynamic_context.block_allocator.allocate_memory_blocks(3, safe=True) dynamic_context.request_to_kv_block_ids[0:3, 0] = new_block_ids + + if is_hybrid_model: + # Also initialize Mamba states for the dummy requests + dynamic_context.mamba_conv_states[:, 0:3, :, :].fill_(1.0) + dynamic_context.mamba_ssm_states[:, 0:3, :, :, :].fill_(1.0) + dynamic_context.update_requests( active_requests_mask=active_requests_mask, new_tokens=torch.tensor([0, 1, 2]) ) assert dynamic_context.total_request_count == 0 + if is_hybrid_model: + assert torch.all(dynamic_context.mamba_conv_states == 0) + assert torch.all(dynamic_context.mamba_ssm_states == 0) # This case would cover all cases # 1. Already there will be 2 paused requests @@ -406,9 +487,9 @@ def test_update_request(self): # 4. Some of these requests will be resumed. # Setup is as follows : # Request ids 0, 1 are paused - # Request ids 2 , 4, 9 are active requests + # Request ids 2, 4, 9 are active requests # Request ids 3 7 8 have completed - # Request ids 5 and 6 will require on more block later on coz they finished their current block + # Request ids 5 and 6 will require on more block later on because they finished their current block dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, @@ -417,11 +498,12 @@ def test_update_request(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) active_requests_mask = torch.Tensor([1, 0, 1, 1, 1, 0, 0, 1]).cuda().int() @@ -472,6 +554,14 @@ def test_update_request(self): dynamic_context.request_last_kv_block_offset[0:2] = dynamic_context.block_size_tokens - 1 dynamic_context.request_last_kv_block_offset[5:7] = dynamic_context.block_size_tokens - 1 + if is_hybrid_model: + # Dummy fill for states to be non-zero before update + for i in range(total_request_count): + dynamic_context.mamba_metadata.request_to_mamba_state_idx[i] = i + dynamic_context.mamba_metadata.mamba_state_free_slot_count -= total_request_count + dynamic_context.mamba_conv_states[:, 0:total_request_count, :, :] = 1.0 + dynamic_context.mamba_ssm_states[:, 0:total_request_count, :, :, :] = 1.0 + dynamic_context.update_requests( active_requests_mask=active_requests_mask, new_tokens=next_tokens ) @@ -522,28 +612,49 @@ def test_update_request(self): # The first 4 requests will require an extra block. # Since 3 requests have finished, the last 3 rows should be all -1. - assert torch.all( - dynamic_context.request_to_kv_block_ids[0:10].cpu() - == torch.tensor( - [ - [479, 482, -1, -1], - [480, 479, -1, -1], - [484, 486, -1, -1], - [485, 487, -1, -1], - [483, -1, -1, -1], - [481, -1, -1, -1], - [488, -1, -1, -1], - [-1, -1, -1, -1], - [-1, -1, -1, -1], - [-1, -1, -1, -1], - ] + if is_hybrid_model: + assert torch.all( + dynamic_context.request_to_kv_block_ids[0:10].cpu() + == torch.tensor( + [ + [1144, 1147, -1, -1], + [1145, 1144, -1, -1], + [1149, 1151, -1, -1], + [1150, 1152, -1, -1], + [1148, -1, -1, -1], + [1146, -1, -1, -1], + [1153, -1, -1, -1], + [-1, -1, -1, -1], + [-1, -1, -1, -1], + [-1, -1, -1, -1], + ] + ) + ) + else: + assert torch.all( + dynamic_context.request_to_kv_block_ids[0:10].cpu() + == torch.tensor( + [ + [479, 482, -1, -1], + [480, 479, -1, -1], + [484, 486, -1, -1], + [485, 487, -1, -1], + [483, -1, -1, -1], + [481, -1, -1, -1], + [488, -1, -1, -1], + [-1, -1, -1, -1], + [-1, -1, -1, -1], + [-1, -1, -1, -1], + ] + ) ) - ) @pytest.mark.internal - def test_release_memory_blocks_for_finished_requests(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_release_memory_blocks_for_finished_requests(self, is_hybrid_model): """Test that memory blocks are correctly released for finished requests.""" self._setup_model_parallel_group(1, 1) + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=4, @@ -551,11 +662,12 @@ def test_release_memory_blocks_for_finished_requests(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) # Set up the initial state with 5 requests @@ -572,6 +684,13 @@ def test_release_memory_blocks_for_finished_requests(self): dynamic_context.request_to_kv_block_ids[i, 0] = initial_blocks[i] dynamic_context.request_query_lengths[i] = 1 dynamic_context.request_ids[i] = i + if is_hybrid_model: + dynamic_context.mamba_conv_states[:, i, :, :].fill_( + float(i + 1) + ) # Fill with distinct values + dynamic_context.mamba_ssm_states[:, i, :, :, :].fill_(float(i + 1)) + dynamic_context.mamba_metadata.request_to_mamba_state_idx[i] = i + dynamic_context.mamba_metadata.mamba_state_free_slot_count -= 1 # Create an active_requests_mask where requests 0, 2, and 4 are finished (0), # and requests 1 and 3 are still active (1) @@ -591,10 +710,26 @@ def test_release_memory_blocks_for_finished_requests(self): # Verify that 3 blocks were released by checking the available blocks assert dynamic_context.block_allocator.block_count_avail == initial_available_blocks + 3 + if is_hybrid_model: + # Request at position 3 now moves into finished request position 0 + # Request at position 1 remains active + mamba_idx = { + i: dynamic_context.mamba_metadata.request_to_mamba_state_idx[i] for i in range(5) + } + assert torch.all(dynamic_context.mamba_conv_states[:, mamba_idx[0], :, :] == 4.0) + assert torch.all(dynamic_context.mamba_ssm_states[:, mamba_idx[0], :, :, :] == 4.0) + assert torch.all(dynamic_context.mamba_conv_states[:, mamba_idx[1], :, :] == 2.0) + assert torch.all(dynamic_context.mamba_ssm_states[:, mamba_idx[1], :, :, :] == 2.0) + assert mamba_idx[2] == -1 + assert mamba_idx[3] == -1 + assert mamba_idx[4] == -1 + @pytest.mark.internal - def test_finished_requests_with_multiple_blocks(self): + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_finished_requests_with_multiple_blocks(self, is_hybrid_model): """Test that all memory blocks are correctly released for finished requests that use multiple blocks.""" self._setup_model_parallel_group(1, 1) + dynamic_context = self._get_dynamic_context( params_dtype=torch.float32, num_layers=4, @@ -602,11 +737,12 @@ def test_finished_requests_with_multiple_blocks(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, ) # Set up the initial state with 3 requests, where some use multiple blocks @@ -638,6 +774,9 @@ def test_finished_requests_with_multiple_blocks(self): for i in range(3): dynamic_context.request_query_lengths[i] = 1 dynamic_context.request_ids[i] = i + if is_hybrid_model: + dynamic_context.mamba_conv_states[:, i, :, :].fill_(float(i + 1)) + dynamic_context.mamba_ssm_states[:, i, :, :, :].fill_(float(i + 1)) # Create an active_requests_mask where all requests are finished active_requests_mask = torch.tensor([0, 0, 0], device=torch.cuda.current_device()) @@ -655,6 +794,92 @@ def test_finished_requests_with_multiple_blocks(self): # Verify that all 6 blocks were released by checking the available blocks assert dynamic_context.block_allocator.block_count_avail == initial_available_blocks + 6 + if is_hybrid_model: + # All mamba states should be zeroed out + assert torch.all(dynamic_context.mamba_conv_states == 0) + assert torch.all(dynamic_context.mamba_ssm_states == 0) + + @pytest.mark.internal + @pytest.mark.parametrize("is_hybrid_model", [False, True]) + def test_mamba_states_cache(self, is_hybrid_model: bool): + self._setup_model_parallel_group(1, 1) + + if not is_hybrid_model: + # If not hybrid, mamba_states_cache should fail + dynamic_context = self._get_dynamic_context( + params_dtype=torch.float32, + num_layers=4, + kv_channels=8, + num_attention_heads=2, + max_sequence_length=512, + buffer_size_gb=0.03, + buffer_guaranteed_fraction=0.1, + block_size_tokens=128, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, + is_hybrid_model=False, + ) + with pytest.raises(AssertionError) as error: + conv_state, ssm_state = dynamic_context.mamba_states_cache(layer_number=1) + return + + dynamic_context = self._get_dynamic_context( + params_dtype=torch.float32, + num_layers=4, + kv_channels=8, + num_attention_heads=2, + max_sequence_length=512, + buffer_size_gb=0.03, + buffer_guaranteed_fraction=0.1, + block_size_tokens=128, + max_requests_override=None, + max_tokens_override=None, + buffer_overflow_factor=None, + is_hybrid_model=is_hybrid_model, + layer_type_list=[Symbols.MAMBA, Symbols.ATTENTION, Symbols.MAMBA, Symbols.ATTENTION], + ) + + # Add a request to populate states + context_length = 10 + dynamic_context.add_request( + DynamicInferenceRequest( + request_id=0, + prompt_tokens=torch.arange(0, context_length, dtype=torch.long, device='cuda'), + sampling_params=SamplingParams( + num_tokens_to_generate=dynamic_context.max_tokens - 10 + ), + ) + ) + dynamic_context.initialize_attention_state() + + # Manually set some dummy values in mamba_conv_states and mamba_ssm_states + # Mamba layers are at global indices 0 and 2 (mapped to local 0 and 1 via layer_map) + # `layer_map` will map global layer index to the corresponding Mamba/Attention index. + # For layer_type_list ["MAMBA", "ATTENTION", "MAMBA", "ATTENTION"], + # global layer 1 (index 0) is MAMBA -> local mamba layer 0 + # global layer 3 (index 2) is MAMBA -> local mamba layer 1 + + # Test for the first Mamba layer (global layer 1, local mamba layer 0) + global_layer_1_mamba_local_idx = 0 + dynamic_context.mamba_conv_states[global_layer_1_mamba_local_idx] = 10.0 + dynamic_context.mamba_ssm_states[global_layer_1_mamba_local_idx] = 20.0 + + # Test for the second Mamba layer (global layer 3, local mamba layer 1) + global_layer_3_mamba_local_idx = 1 + dynamic_context.mamba_conv_states[global_layer_3_mamba_local_idx] = 30.0 + dynamic_context.mamba_ssm_states[global_layer_3_mamba_local_idx] = 40.0 + + # Retrieve states using mamba_states_cache for global layer 1 + conv_state_layer1, ssm_state_layer1 = dynamic_context.mamba_states_cache(layer_number=1) + assert torch.all(conv_state_layer1 == 10.0) + assert torch.all(ssm_state_layer1 == 20.0) + + # Retrieve states using mamba_states_cache for global layer 3 + conv_state_layer3, ssm_state_layer3 = dynamic_context.mamba_states_cache(layer_number=3) + assert torch.all(conv_state_layer3 == 30.0) + assert torch.all(ssm_state_layer3 == 40.0) + @pytest.mark.internal def test_calculate_and_store_log_probs(self): self._setup_model_parallel_group(1, 1) @@ -665,7 +890,7 @@ def test_calculate_and_store_log_probs(self): num_attention_heads=2, max_sequence_length=512, buffer_size_gb=0.03, - buffer_guarenteed_fraction=0.1, + buffer_guaranteed_fraction=0.1, block_size_tokens=128, max_requests_override=None, max_tokens_override=None, diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine.py b/tests/unit_tests/inference/engines/test_dynamic_engine.py index bd8f0ae1c8a..b019c8d9528 100644 --- a/tests/unit_tests/inference/engines/test_dynamic_engine.py +++ b/tests/unit_tests/inference/engines/test_dynamic_engine.py @@ -4,11 +4,12 @@ import random import types from dataclasses import dataclass -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import pytest import torch from tqdm import tqdm +from transformer_engine.pytorch.fp8 import check_fp8_support from megatron.core import parallel_state from megatron.core.inference.contexts.dynamic_context import ( @@ -31,15 +32,34 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) -from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_local_spec +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, +) from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec +from megatron.core.models.mamba.mamba_model import MambaModel from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from megatron.core.transformer.cuda_graphs import CudaGraphManager, _CudagraphGlobalRecord from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.utils import is_fa_min_version +from megatron.core.utils import ( + check_mamba_sequence_packing_support, + get_attr_wrapped_model, + is_fa_min_version, + is_te_min_version, +) from tests.unit_tests.test_utilities import Utils +def skip_if_mamba_sequence_packing_not_available(model_provider: str): + if model_provider == "mamba": + sequence_packing_available, reason_for_no_sequence_packing = ( + check_mamba_sequence_packing_support() + ) + if not sequence_packing_available: + pytest.skip(reason_for_no_sequence_packing) + + def set_rounder(value): """Utility function to set the DynamicInferenceContext rounder.""" DynamicInferenceContext.ROUNDER = value # For backwards compatibility @@ -77,6 +97,9 @@ class DynamicEngineTestConfig: use_fixed_output_lengths: bool = False num_cuda_graphs: int = None + use_cuda_graphs_for_non_decode_steps: bool = True + fp8: bool = False + model_provider: str = "gpt" return_log_probs: bool = False materialize_only_last_token_logits: bool = True skip_prompt_log_probs_for_dynamic_inference: bool = False @@ -89,6 +112,8 @@ class DynamicEngineTestConfig: # relevant to the test. The tests only check if the required # context attributes are set correctly. + fp8: bool = False + def __post_init__(self): # Compute max_sequence_length. @@ -161,6 +186,7 @@ def _build_requests(cls, test_config: DynamicEngineTestConfig) -> List[DynamicIn -1 if test_config.use_fixed_output_lengths else test_config.vocab_size - 1 ), return_log_probs=test_config.return_log_probs, + skip_prompt_log_probs=test_config.skip_prompt_log_probs, ) if not hasattr(sampling_params, "num_tokens_total"): # Remove this if statement branch in megatron-core 0.16 @@ -168,11 +194,6 @@ def _build_requests(cls, test_config: DynamicEngineTestConfig) -> List[DynamicIn else: sampling_params.num_tokens_total = num_tokens_total - config_entry = test_config.skip_prompt_log_probs_for_dynamic_inference - sampling_params.add_attributes( - {"skip_prompt_log_probs_for_dynamic_inference": config_entry} - ) - # Request. prompt_tokens = torch.randint( 0, @@ -194,6 +215,9 @@ def _build_inference_context( test_config: DynamicEngineTestConfig, transformer_config: TransformerConfig, requests: List[DynamicInferenceRequest], + layer_type_list: Optional[List[str]], + mamba_conv_states_shape: Optional[Tuple[int]] = None, + mamba_ssm_states_shape: Optional[Tuple[int]] = None, ): """The inference context manages the KV cache and other inference state.""" @@ -205,6 +229,7 @@ def _build_inference_context( num_attention_heads=transformer_config.num_query_groups, max_sequence_length=test_config.max_sequence_length, num_cuda_graphs=test_config.num_cuda_graphs, + use_cuda_graphs_for_non_decode_steps=not test_config.model_provider == "mamba", buffer_size_gb=test_config.context_buffer_size_gb, buffer_guaranteed_fraction=test_config.context_buffer_guaranteed_fraction, block_size_tokens=test_config.context_block_size_tokens, @@ -212,6 +237,9 @@ def _build_inference_context( max_requests_override=test_config.context_max_requests_override, max_tokens_override=test_config.context_max_tokens_override, tensor_model_parallel_size=transformer_config.tensor_model_parallel_size, + layer_type_list=layer_type_list, + mamba_conv_states_shape=mamba_conv_states_shape, + mamba_ssm_states_shape=mamba_ssm_states_shape, materialize_only_last_token_logits=test_config.materialize_only_last_token_logits, use_flashinfer_fused_rope=None, # default to using flash-infer if available # this is for compatibility with the LTS environment @@ -220,6 +248,7 @@ def _build_inference_context( return context @classmethod + @torch.inference_mode() def _build_test_env(cls, test_config): Utils.initialize_model_parallel( tensor_model_parallel_size=test_config.tensor_model_parallel_size, @@ -238,65 +267,142 @@ def _build_test_env(cls, test_config): force_reset_rng=True, ) - # Transformer config. - transformer_config = TransformerConfig( - params_dtype=torch.bfloat16, - num_layers=4, - hidden_size=32, - num_attention_heads=4, - use_cpu_initialization=True, - cuda_graph_impl=( - "local" - if test_config.num_cuda_graphs is not None and test_config.force_build_cuda_graphs - else "none" - ), - inference_rng_tracker=True, - tensor_model_parallel_size=test_config.tensor_model_parallel_size, - pipeline_model_parallel_size=test_config.pipeline_model_parallel_size, - expert_model_parallel_size=test_config.expert_model_parallel_size, - num_moe_experts=( - None - if test_config.expert_model_parallel_size == 1 - else test_config.expert_model_parallel_size - ), - sequence_parallel=test_config.sequence_parallel, - pipeline_dtype=torch.bfloat16, - add_bias_linear=test_config.expert_model_parallel_size == 1, - inference_sampling_seed=test_config.random_seed, - cuda_graph_scope=test_config.cuda_graph_scope, - ) - # Requests. requests = cls._build_requests(test_config) - # GPT model. - model = GPTModel( - config=transformer_config, - transformer_layer_spec=get_gpt_layer_local_spec(), - vocab_size=test_config.vocab_size, - max_sequence_length=test_config.max_sequence_length, - parallel_output=True, - pre_process=parallel_state.is_pipeline_first_stage(), - post_process=parallel_state.is_pipeline_last_stage(), - ).cuda() + if test_config.model_provider == "gpt": + # Transformer config. + transformer_config = TransformerConfig( + params_dtype=torch.bfloat16, + num_layers=4, + hidden_size=128 if test_config.fp8 else 32, + num_attention_heads=4, + use_cpu_initialization=True, + cuda_graph_impl=( + "local" + if test_config.num_cuda_graphs is not None + and test_config.force_build_cuda_graphs + else "none" + ), + inference_rng_tracker=True, + tensor_model_parallel_size=test_config.tensor_model_parallel_size, + pipeline_model_parallel_size=test_config.pipeline_model_parallel_size, + expert_model_parallel_size=test_config.expert_model_parallel_size, + num_moe_experts=( + None + if test_config.expert_model_parallel_size == 1 + else test_config.expert_model_parallel_size + ), + sequence_parallel=test_config.sequence_parallel, + pipeline_dtype=torch.bfloat16, + add_bias_linear=test_config.expert_model_parallel_size == 1, + fp8="hybrid" if test_config.fp8 else None, + fp8_recipe="tensorwise" if test_config.fp8 else None, + inference_sampling_seed=test_config.random_seed, + cuda_graph_scope=test_config.cuda_graph_scope, + ) + if test_config.fp8: + layer_spec = get_gpt_layer_with_transformer_engine_spec() + else: + layer_spec = get_gpt_layer_local_spec() + + # GPT model. + model = GPTModel( + config=transformer_config, + transformer_layer_spec=layer_spec, + vocab_size=test_config.vocab_size, + max_sequence_length=test_config.max_sequence_length, + parallel_output=True, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ).cuda() + elif test_config.model_provider == "mamba": + # Transformer config. + transformer_config = TransformerConfig( + params_dtype=torch.bfloat16, + num_layers=3, # 1 Mamba layer, 1 attention layer, 1 MLP layer + hidden_size=256, # The Mamba layer places several constraints on this + mamba_num_heads=16, + num_attention_heads=16, + use_cpu_initialization=True, + cuda_graph_impl=( + "local" + if test_config.num_cuda_graphs is not None + and test_config.force_build_cuda_graphs + else "none" + ), + inference_rng_tracker=True, + tensor_model_parallel_size=test_config.tensor_model_parallel_size, + pipeline_model_parallel_size=test_config.pipeline_model_parallel_size, + expert_model_parallel_size=test_config.expert_model_parallel_size, + num_moe_experts=( + None + if test_config.expert_model_parallel_size == 1 + else test_config.expert_model_parallel_size + ), + sequence_parallel=test_config.sequence_parallel, + pipeline_dtype=torch.bfloat16, + add_bias_linear=test_config.expert_model_parallel_size == 1, + fp8="hybrid" if test_config.fp8 else None, + fp8_recipe="tensorwise" if test_config.fp8 else None, + cuda_graph_scope=test_config.cuda_graph_scope, + ) + + # Mamba model. + model = MambaModel( + config=transformer_config, + mamba_stack_spec=mamba_stack_spec, + vocab_size=test_config.vocab_size, + max_sequence_length=test_config.max_sequence_length, + parallel_output=True, + hybrid_attention_ratio=0.3, + hybrid_mlp_ratio=0.3, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ).cuda() + else: + raise ValueError(f"Invalid model provider {test_config.model_provider}") for param in model.parameters(): param.data = param.data.to(transformer_config.params_dtype) model.eval() + # Layer type list for hybrid models + decoder = get_attr_wrapped_model(model, "decoder") + layer_type_list = getattr(decoder, "layer_type_list", None) + if test_config.model_provider == "mamba": + mamba_states_shapes = decoder.mamba_state_shapes_per_request() + if mamba_states_shapes is not None: + (mamba_conv_states_shape, mamba_ssm_states_shape) = mamba_states_shapes + else: + # A `MambaBlock` can only not have a `MambaLayer` if using pipeline parallelism + # and a particular pipeline stage was not assigned a `MambaLayer`. + assert test_config.pipeline_model_parallel_size > 1 + mamba_conv_states_shape = None + mamba_ssm_states_shape = None + else: + mamba_conv_states_shape = None + mamba_ssm_states_shape = None + # Inference config. inference_config = InferenceWrapperConfig( hidden_size=transformer_config.hidden_size, inference_batch_times_seqlen_threshold=400, fp32_residual_connection=False, params_dtype=transformer_config.params_dtype, + fp8=transformer_config.fp8, padded_vocab_size=test_config.vocab_size, ) # Inference context. inference_context = cls._build_inference_context( - test_config=test_config, transformer_config=transformer_config, requests=requests + test_config=test_config, + transformer_config=transformer_config, + requests=requests, + layer_type_list=layer_type_list, + mamba_conv_states_shape=mamba_conv_states_shape, + mamba_ssm_states_shape=mamba_ssm_states_shape, ) # Inference model wrapper. @@ -338,6 +444,7 @@ def mock_detokenize_prompt(tokens): return env @classmethod + @torch.inference_mode() def _run_step(cls, env): set_rounder(4) # Step inference engine (i.e., generate one token per request). @@ -349,8 +456,8 @@ def _run_step(cls, env): finished_requests = result["finished_requests"] @classmethod + @torch.inference_mode() def _run_test(cls, **test_config_kwargs): - # Test environment. test_config = DynamicEngineTestConfig(**test_config_kwargs) env = cls._build_test_env(test_config) @@ -405,13 +512,16 @@ def teardown_method(self, method): @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) @pytest.mark.parametrize("num_cuda_graphs", [None, 1, 4]) @pytest.mark.parametrize("cuda_graph_scope", [[], ["full_iteration"]]) - def test_simple(self, num_cuda_graphs, cuda_graph_scope) -> None: + def test_simple(self, model_provider, num_cuda_graphs, cuda_graph_scope) -> None: """Simple test that runs without errors, and validates output.""" + skip_if_mamba_sequence_packing_not_available(model_provider) # Run test. env = self._run_test( + model_provider=model_provider, num_cuda_graphs=num_cuda_graphs, context_max_requests_override=32, cuda_graph_scope=cuda_graph_scope, @@ -422,8 +532,8 @@ def test_simple(self, num_cuda_graphs, cuda_graph_scope) -> None: assert env.engine.context.max_requests == 32 assert env.engine.context.max_tokens == 160 - # Validate generated tokens. - expected_generated_tokens_list = [ + # Validate output tokens. + gpt_expected_generated_tokens = [ [69, 85, 55, 74], [29, 54, 85, 89], [33, 30, 64, 59], @@ -434,7 +544,26 @@ def test_simple(self, num_cuda_graphs, cuda_graph_scope) -> None: [], # this request is failed due to max sequence length overflow ] + mamba_expected_generated_tokens = [ + [74, 72, 83, 59], + [25, 54, 1, 70], + [28, 14, 15, 89], + [87, 27, 30, 52], + [44, 13, 82, 70], + [28, 74, 64, 16], + [8, 4, 83, 5], + [], + ] + + if model_provider == "gpt": + expected_generated_tokens_list = gpt_expected_generated_tokens + elif model_provider == "mamba": + expected_generated_tokens_list = mamba_expected_generated_tokens + else: + raise ValueError(f"Invalid model_provider {model_provider}") + assert len(env.requests) == len(expected_generated_tokens_list) + for request, expected_generated_tokens in zip(env.requests, expected_generated_tokens_list): assert request.generated_tokens == expected_generated_tokens, ( f"request {request.request_id}, " @@ -446,30 +575,41 @@ def test_simple(self, num_cuda_graphs, cuda_graph_scope) -> None: @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_overflow_factor(self) -> None: + def test_overflow_factor(self, model_provider: str = "gpt") -> None: """Test overflow factor arg.""" + skip_if_mamba_sequence_packing_not_available(model_provider) + # Run test. env = self._run_test( context_buffer_overflow_factor=0.1, context_max_requests_override=None, context_max_tokens_override=None, + model_provider=model_provider, ) # Validate max_requests, max_tokens. - assert env.engine.context.max_requests == 420 - assert env.engine.context.max_tokens == 420 + if model_provider == "gpt": + assert env.engine.context.max_requests == 420 + assert env.engine.context.max_tokens == 420 + elif model_provider == "mamba": + assert env.engine.context.max_requests == 16 + assert env.engine.context.max_tokens == 16 @pytest.mark.internal @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_request_overflow(self) -> None: + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + def test_request_overflow(self, model_provider: str) -> None: """Test request overflow.""" - self._run_test(context_max_requests_override=4) + skip_if_mamba_sequence_packing_not_available(model_provider) + + self._run_test(context_max_requests_override=4, model_provider=model_provider) @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) + @torch.inference_mode() def test_token_overflow_transient(self) -> None: """Test token overflow.""" test_config = DynamicEngineTestConfig( @@ -505,13 +645,17 @@ def test_token_overflow_nontransient(self) -> None: @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_block_overflow(self) -> None: + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + def test_block_overflow(self, model_provider: str) -> None: """Test block overflow.""" - env = self._build_test_env(DynamicEngineTestConfig()) + skip_if_mamba_sequence_packing_not_available(model_provider) + env = self._build_test_env(DynamicEngineTestConfig(model_provider=model_provider)) context = env.engine.context block_size_bytes = context.block_size_bytes buffer_size_gb = (block_size_bytes + 1) / 1024**3 - test_config = DynamicEngineTestConfig(context_buffer_size_gb=buffer_size_gb) + test_config = DynamicEngineTestConfig( + context_buffer_size_gb=buffer_size_gb, model_provider=model_provider + ) env = self._build_test_env(test_config) env.engine._add_request(env.requests[0]) assert list(env.engine.waiting_request_ids) == [0] @@ -520,17 +664,21 @@ def test_block_overflow(self) -> None: @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_multi_add(self) -> None: + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + def test_multi_add(self, model_provider: str) -> None: """Test adding multiple requests simultaneously.""" - self._run_test(num_gap_steps=0) + skip_if_mamba_sequence_packing_not_available(model_provider) + self._run_test(num_gap_steps=0, model_provider=model_provider) @pytest.mark.internal @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_fixed_output_lengths(self) -> None: + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + def test_fixed_output_lengths(self, model_provider: str) -> None: """Test generating a fixed number of output tokens.""" - self._run_test(use_fixed_output_lengths=True) + skip_if_mamba_sequence_packing_not_available(model_provider) + self._run_test(use_fixed_output_lengths=True, model_provider=model_provider) @pytest.mark.internal @pytest.mark.skipif( @@ -587,6 +735,7 @@ def test_cuda_graph_token_counts(self) -> None: (32, 32), ], ) + @torch.inference_mode() def test_cuda_graph_warmup( self, warmup_engine_mode: WarmupEngineMode, @@ -673,11 +822,18 @@ def test_cuda_graph_warmup( @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) - def test_generate_function(self) -> None: + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + @torch.inference_mode() + def test_generate_function(self, model_provider: str) -> None: """Test the generate function that processes multiple prompts at once.""" + skip_if_mamba_sequence_packing_not_available(model_provider) + # Set up test environment test_config = DynamicEngineTestConfig( - num_requests=4, max_prompt_length=8, num_tokens_to_generate=4 + num_requests=4, + max_prompt_length=8, + num_tokens_to_generate=4, + model_provider=model_provider, ) env = self._build_test_env(test_config) @@ -723,36 +879,55 @@ async def test_run_engine(self): Test asynchronously adding and waiting for requests while the engine is running continuously. """ - # Test environment. - test_config = DynamicEngineTestConfig(use_fixed_output_lengths=True) - env = self._build_test_env(test_config) + # Have to wrap inference mode in-line because async functions are not supported + with torch.inference_mode(): + # Test environment. + test_config = DynamicEngineTestConfig(num_requests=8, use_fixed_output_lengths=True) + env = self._build_test_env(test_config) + + engine_task = asyncio.create_task(env.engine.run_engine(verbose=False)) + + request_completion_futures: Dict[int, asyncio.Future[DynamicInferenceRequest]] = {} + + # Add requests to engine. + for request in tqdm(env.requests, "add requests"): + request_completion_futures[request.request_id] = env.engine._add_request(request) + + # Wait for all requests to complete. + await asyncio.gather(*request_completion_futures.values()) + + # Verify that all request outputs were set. + for request_id, fut in request_completion_futures.items(): + num_tokens_to_generate = env.requests[ + request_id + ].sampling_params.num_tokens_to_generate + result = fut.result() + assert result.generated_length == num_tokens_to_generate, ( + f"Request {request_id} expected to generate {num_tokens_to_generate} " + f"tokens but generated {result.generated_length}" + ) - engine_task = asyncio.create_task(env.engine.run_engine(verbose=False)) + engine_task.cancel() - request_completion_futures: Dict[int, asyncio.Future[DynamicInferenceRequest]] = {} + @pytest.mark.internal + @pytest.mark.skipif( + not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" + ) + @pytest.mark.skipif(not is_te_min_version("2.2.0"), reason="TE 2.2.0 is required") + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + def test_fp8_inference(self, model_provider: str): + skip_if_mamba_sequence_packing_not_available(model_provider) - # Add requests to engine. - for request in tqdm(env.requests, "add requests"): - request_completion_futures[request.request_id] = env.engine._add_request(request) - - # Wait for all requests to complete. - await asyncio.gather(*request_completion_futures.values()) - - # Verify that all request outputs were set. - for request_id, fut in request_completion_futures.items(): - num_tokens_to_generate = env.requests[request_id].sampling_params.num_tokens_to_generate - result = fut.result() - assert result.generated_length == num_tokens_to_generate, ( - f"Request {request_id} expected to generate {num_tokens_to_generate} " - f"tokens but generated {result.generated_length}" - ) + fp8_available, reason_for_no_fp8 = check_fp8_support() + if not fp8_available: + pytest.skip(reason_for_no_fp8) - engine_task.cancel() + self._run_test(model_provider=model_provider, fp8=True) - @pytest.mark.internal @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" ) + @torch.inference_mode() def test_return_log_probs(self): """Verify that returning log probs does not raise any error.""" # Returning log probs requires materializing the full prompt logits or @@ -763,7 +938,7 @@ def test_return_log_probs(self): env = self._run_test( return_log_probs=True, materialize_only_last_token_logits=True, - skip_prompt_log_probs_for_dynamic_inference=True, + skip_prompt_log_probs=True, ) @pytest.mark.internal @@ -775,9 +950,19 @@ def test_return_log_probs(self): @pytest.mark.parametrize("ep_size", [1, 2]) @pytest.mark.parametrize("pp_size", [1, 2]) @pytest.mark.parametrize("tp_size", [1, 2]) + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + @torch.inference_mode() def test_parallel_inference( - self, tp_size, pp_size, ep_size, sequence_parallel, materialize_only_last_token_logits + self, + model_provider, + tp_size, + pp_size, + ep_size, + sequence_parallel, + materialize_only_last_token_logits, ): + skip_if_mamba_sequence_packing_not_available(model_provider) + if tp_size == 1 and pp_size == 1 and ep_size == 1: pytest.skip(reason="Test requires tp_size > 1 or pp_size > 1 or ep_size > 1") elif not torch.distributed.is_initialized(): @@ -790,7 +975,16 @@ def test_parallel_inference( pytest.skip(reason="Sequence parallelism requires tp_size > 1") elif tp_size > 1 and ep_size > 1 and not sequence_parallel: pytest.skip(reason="Sequence parallelism must be used with tp_size > 1 and ep_size > 1") + elif pp_size > 1 and model_provider == "mamba": + pytest.skip( + reason=( + "Running hybrid models with pp_size > 1 and no attention on some " + "pipeline stages is not supported yet." + ) + ) + env = self._run_test( + model_provider=model_provider, tensor_model_parallel_size=tp_size, pipeline_model_parallel_size=pp_size, expert_model_parallel_size=ep_size, @@ -798,6 +992,25 @@ def test_parallel_inference( materialize_only_last_token_logits=materialize_only_last_token_logits, ) + @pytest.mark.internal + @pytest.mark.skipif( + not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" + ) + @pytest.mark.parametrize("materialize_only_last_token_logits", [False, True]) + def test_sequence_parallel_fp8_inference(self, materialize_only_last_token_logits: bool): + fp8_available, reason_for_no_fp8 = check_fp8_support() + if not fp8_available: + pytest.skip(reason_for_no_fp8) + + self._run_test( + min_prompt_length=19, + max_prompt_length=19, + tensor_model_parallel_size=4, + sequence_parallel=True, + materialize_only_last_token_logits=True, + fp8=True, + ) + @pytest.mark.internal @pytest.mark.skipif( not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" @@ -852,6 +1065,32 @@ def test_events(self): assert result_event_types == expected_event_types + @pytest.mark.internal + @pytest.mark.skipif( + not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" + ) + @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) + @torch.inference_mode() + def test_chunked_prefill(self, model_provider: str): + """Verify that chunked prefill output is equivalent to regular prefill.""" + skip_if_mamba_sequence_packing_not_available(model_provider) + + prompt_length = 1200 + num_tokens_to_generate = 16 + max_sequence_length = prompt_length + num_tokens_to_generate + + # Configure context to force chunking (chunked prefill is enabled by default) + env = self._run_test( + num_requests=1, + min_prompt_length=prompt_length, + max_prompt_length=prompt_length, + num_tokens_to_generate=num_tokens_to_generate, + materialize_only_last_token_logits=False, + model_provider=model_provider, + context_block_size_tokens=256, + context_max_tokens_override=300, + ) + if __name__ == "__main__": test = TestDynamicInferenceEngine() diff --git a/tests/unit_tests/inference/model_inference_wrappers/gpt/test_gpt_inference_wrapper.py b/tests/unit_tests/inference/model_inference_wrappers/gpt/test_gpt_inference_wrapper.py index 644cb149988..07afebe1067 100644 --- a/tests/unit_tests/inference/model_inference_wrappers/gpt/test_gpt_inference_wrapper.py +++ b/tests/unit_tests/inference/model_inference_wrappers/gpt/test_gpt_inference_wrapper.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + from argparse import Namespace import pytest @@ -32,7 +34,7 @@ def setup_model(self, tensor_parallel_size, pipeline_parallel_size): self.vocab_size = 100 self.batch_size = 4 self.sequence_length = 32 - hidden_size = 12 + hidden_size = 32 transformer_config = TransformerConfig( num_layers=4, diff --git a/tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py b/tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py index f23a9782646..10ffe2fdd40 100644 --- a/tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py +++ b/tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py @@ -66,7 +66,7 @@ def setup_model( else: model_parallel_cuda_manual_seed(123, inference_rng_tracker=True) self.batch_size = batch_size - self.hidden_size = 12 + self.hidden_size = 32 self.vocab_size = 100 self.sequence_length = 60 if fp8 else 64 # Test padding for fp8 transformer_config = TransformerConfig( diff --git a/tools/run_inference_performance_test.py b/tools/run_inference_performance_test.py index 2f2adabc0ab..01e5ab58898 100644 --- a/tools/run_inference_performance_test.py +++ b/tools/run_inference_performance_test.py @@ -1,83 +1,59 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -import os -from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import ( - InferenceWrapperConfig, -) import argparse +import os import random -import torch import sys import time -import tqdm -import warnings -from model_provider import model_provider + +import torch + from gpt_builders import gpt_builder from mamba_builders import mamba_builder -from megatron.core.inference.engines.abstract_engine import AbstractEngine +from megatron.core.inference.contexts import DynamicInferenceContext from megatron.core.inference.engines import DynamicInferenceEngine, StaticInferenceEngine +from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.inference_request import InferenceRequest -from megatron.core.inference.contexts import DynamicInferenceContext -from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( GPTInferenceWrapper, ) +from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import ( + InferenceWrapperConfig, +) +from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.ssm.mamba_hybrid_layer_allocation import Symbols from megatron.core.transformer.module import MegatronModule +from megatron.core.utils import get_attr_wrapped_model +from model_provider import model_provider sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) ) -from megatron.training import get_args -from megatron.training import get_tokenizer -from megatron.training.checkpointing import load_checkpoint -from megatron.core import mpu -from megatron.training.initialize import initialize_megatron -from megatron.training import get_model, get_tokenizer import asyncio from functools import partial -from typing import AsyncIterator, List, Union +from typing import List, Union + +from examples.inference.gpt.utils import add_common_inference_args +from megatron.core import mpu +from megatron.training import get_args, get_model, get_tokenizer +from megatron.training.checkpointing import load_checkpoint +from megatron.training.initialize import initialize_megatron REQUEST_ID = 0 -def add_text_generate_args(parser): - """Text generation arguments.""" - group = parser.add_argument_group(title='text generation') +def add_inference_benchmarking_args(parser): + """Inference benchmarking arguments.""" + parser = add_common_inference_args(parser) + + group = parser.add_argument_group(title='inference_benchmarking') - group.add_argument("--temperature", type=float, default=1.0, help='Sampling temperature.') - group.add_argument("--top_k", type=int, default=1, help='Top k sampling.') - group.add_argument("--top_p", type=float, default=0.0, help='Top p sampling.') - group.add_argument( - "--return-log-probs", - action='store_true', - default=False, - help='Return the log probabilities of the final output tokens', - ) - group.add_argument("--top-n-logprobs", type=int, default=0, help="Top-N logprobs") - group.add_argument( - "--num-tokens-to-generate", - type=int, - default=30, - help='Number of tokens to generate for each prompt', - ) - group.add_argument( - "--prompts", - metavar='N', - type=str, - default=None, - nargs='+', - help='Input prompts with each prompt within quotes and seperated by space', - ) - group.add_argument( - "--num-input-tokens", type=int, default=None, help='Number of input tokens per prompt' - ) - group.add_argument("--stream", action="store_true", default=False, help="Stream output tokens") group.add_argument( - "--model-provider", choices=["mamba", "gpt"], default="gpt", help="Model provider" + "--num-input-tokens", type=int, default=128, help="Number of input tokens per request" ) group.add_argument( "--engine-type", choices=["static", "dynamic"], default="static", help="Engine type" @@ -85,14 +61,13 @@ def add_text_generate_args(parser): group.add_argument( "--benchmark-profile", action="store_true", default=False, help="If set, profile" ) + group.add_argument('--stream', action="store_true", default=False, help="If set, stream tokens") return parser def get_inference_engine(args: argparse.Namespace, model: MegatronModule) -> AbstractEngine: """Utility to get the relevant backend for running inference - This function will automatically chose the TRTLLMBackend when possible, and if not revert to Mcore backend if the user does not specify any backends. TRT LLM Backend is not implmented yet. - Args: args (Namespace): The user arguments parsed from command line model (MegatronModule): The megatron model . @@ -111,9 +86,18 @@ def get_inference_engine(args: argparse.Namespace, model: MegatronModule) -> Abs inference_max_requests=args.inference_max_batch_size, inference_max_seq_length=args.inference_max_seq_length, nccl_all_reduce_for_prefill=args.nccl_all_reduce_for_prefill, - moe_pad_experts_for_cuda_graph_inference = args.moe_pad_experts_for_cuda_graph_inference + moe_pad_experts_for_cuda_graph_inference=args.moe_pad_experts_for_cuda_graph_inference, ) + # Layer type list for hybrid models + decoder = get_attr_wrapped_model(model, "decoder") + layer_type_list = getattr(decoder, "layer_type_list", None) + if layer_type_list is not None and Symbols.MAMBA in layer_type_list: + (mamba_conv_states_shape, mamba_ssm_states_shape) = decoder.mamba_state_shapes_per_request() + else: + mamba_conv_states_shape = None + mamba_ssm_states_shape = None + if args.engine_type == "static": inference_wrapped_model = GPTInferenceWrapper(model, inference_wrapper_config) inference_wrapped_model.model_is_pipeline_parallel = not ( @@ -132,12 +116,28 @@ def get_inference_engine(args: argparse.Namespace, model: MegatronModule) -> Abs args.num_query_groups if args.group_query_attention else args.num_attention_heads ), max_sequence_length=args.inference_max_seq_length, + num_cuda_graphs=( + args.inference_dynamic_batching_num_cuda_graphs + if args.cuda_graph_impl == "local" + else None + ), buffer_size_gb=args.inference_dynamic_batching_buffer_size_gb, buffer_guaranteed_fraction=args.inference_dynamic_batching_buffer_guaranteed_fraction, buffer_overflow_factor=args.inference_dynamic_batching_buffer_overflow_factor, max_requests_override=args.inference_dynamic_batching_max_requests_override, max_tokens_override=args.inference_dynamic_batching_max_tokens_override, block_size_tokens=args.inference_dynamic_batching_block_size, + tensor_model_parallel_size=args.tensor_model_parallel_size, + materialize_only_last_token_logits=not args.return_log_probs, + layer_type_list=layer_type_list, + mamba_conv_states_shape=mamba_conv_states_shape, + mamba_ssm_states_shape=mamba_ssm_states_shape, + cache_mla_latent=args.multi_latent_attention and args.cache_mla_latents, + kv_lora_rank=args.kv_lora_rank if args.multi_latent_attention else None, + qk_pos_emb_head_dim=args.qk_pos_emb_head_dim, + use_cuda_graphs_for_non_decode_steps=not args.decode_only_cuda_graphs, + use_flashinfer_fused_rope=args.use_flashinfer_fused_rope, + unified_memory_level=args.inference_dynamic_batching_unified_memory_level, ) inference_wrapped_model = GPTInferenceWrapper( model, inference_wrapper_config, inference_context=context @@ -269,7 +269,7 @@ def main(): # Note: The default args passed here can be overwritten by using appropriate params (check arguments.py file) # Micro batch size is not needed to be set by user. (It is calculated based on inference-batch-times-seqlen-threshold argument) initialize_megatron( - extra_args_provider=add_text_generate_args, + extra_args_provider=add_inference_benchmarking_args, args_defaults={ 'no_load_rng': True, 'no_load_optim': True, @@ -285,6 +285,8 @@ def main(): model_builder = gpt_builder elif args.model_provider == "mamba": model_builder = mamba_builder + else: + raise ValueError(f"Invalid model provider {args.model_provider}") model = get_model(partial(model_provider, model_builder), wrap_with_ddp=False) tokenizer = get_tokenizer() @@ -338,10 +340,7 @@ def main(): print(f"Running warmup for CUDA graphs...") warmup_sampling_params = SamplingParams(num_tokens_to_generate=10) warmup_sampling_params.add_attributes({"no_early_termination": True}) - if args.engine_type == "static": - inference_engine.generate(prompts=["warmup"], sampling_params=warmup_sampling_params) - elif args.engine_type == "dynamic": - generate_dynamic(args, requests, inference_engine) + inference_engine.generate(prompts=["warmup"], sampling_params=warmup_sampling_params) if args.benchmark_profile: torch.cuda.cudart().cudaProfilerStart() diff --git a/tools/run_text_generation_server.py b/tools/run_text_generation_server.py index fb5212f7649..350173dc16f 100644 --- a/tools/run_text_generation_server.py +++ b/tools/run_text_generation_server.py @@ -32,6 +32,7 @@ from megatron.core.inference.text_generation_server import MegatronServer from megatron.core.inference.text_generation_server.run_mcore_engine import run_mcore_engine from megatron.core.transformer.module import MegatronModule +from megatron.post_training.arguments import add_modelopt_args from megatron.training import get_model, print_rank_0 from model_provider import model_provider @@ -120,6 +121,7 @@ def add_text_generate_args(parser): default=None, help='Deprecated in favor of `--inference-max-batch-size`', ) + add_modelopt_args(parser) return parser