diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 95a5dfc86c8..eadc35b9b82 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -232,11 +232,26 @@ jobs: with: python-version: 3.12 + - name: Install GH CLI + shell: bash -x -e -u -o pipefail {0} + run: | + apt-get update + apt-get install -y gh + - name: Get PR info id: get-pr-info if: startsWith(github.ref, 'refs/heads/pull-request/') uses: nv-gha-runners/get-pr-info@main + - name: Has lts label + id: has-lts-label + env: + GH_TOKEN: ${{ secrets.PAT }} + run: | + PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} + HAS_LTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "container::lts")') || echo "false" + echo "main=$HAS_LTS_LABEL" | tee -a $GITHUB_OUTPUT + - name: Download test data shell: bash env: @@ -275,6 +290,22 @@ jobs: echo "$LAST_PRS" | tee -a $GITHUB_OUTPUT echo "EOF" | tee -a $GITHUB_OUTPUT + - name: Parse baseimage + shell: bash + id: base-image + env: + HAS_LTS_LABEL: ${{ steps.has-lts-label.outputs.main }} + run: | + if [ "$HAS_LTS_LABEL" == "true" ]; then + NGC_VERSION=$(cat docker/.ngc_version.lts) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=lts" | tee -a $GITHUB_OUTPUT + else + NGC_VERSION=$(cat docker/.ngc_version.dev) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=dev" | tee -a $GITHUB_OUTPUT + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -286,7 +317,8 @@ jobs: context: . target: main build-args: | - FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:25.09-py3 + FROM_IMAGE_NAME=${{ steps.base-image.outputs.version }} + IMAGE_TYPE=${{ steps.base-image.outputs.image_type }} cache-from: | type=registry,ref=${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}-buildcache,mode=max type=registry,ref=${{ env.container-registry }}/megatron-lm:main-buildcache,mode=max diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index 0658daaa9ec..d67225311f6 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -53,10 +53,12 @@ test:build_image: parallel: matrix: - IMAGE: CI_MCORE_LTS_IMAGE - FILE: Dockerfile.ci.lts - BASE_IMAGE: nvcr.io/nvidia/pytorch:24.01-py3 + FILE: Dockerfile.ci.dev + IMAGE_TYPE: lts + BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev + IMAGE_TYPE: dev BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.dev @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/.ngc_version.lts b/docker/.ngc_version.lts new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.lts @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index 179c2aa21a6..709583d61fc 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -1,8 +1,6 @@ # syntax=docker/dockerfile:1.3-labs ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - FROM ${FROM_IMAGE_NAME} as main ENV PIP_CONSTRAINT="" ENV DEBIAN_FRONTEND=noninteractive @@ -25,16 +23,16 @@ RUN bash -ex <<"EOF" curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh EOF -ARG WHEEL_DIR COPY README.md pyproject.toml uv.lock /workspace/ COPY megatron/core/__init__.py /workspace/megatron/core/ COPY megatron/core/package_info.py /workspace/megatron/core/ +ARG IMAGE_TYPE=dev RUN --mount=type=cache,target=/root/.cache/uv \ bash -ex <<"EOF" export NVTE_CUDA_ARCHS="80;90;100" uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages uv sync --only-group build - uv sync --extra dev --extra mlm --link-mode copy --locked \ + uv sync --extra ${IMAGE_TYPE} --extra mlm --link-mode copy --locked \ --no-install-package torch \ --no-install-package torchvision \ --no-install-package triton \ diff --git a/docker/Dockerfile.ci.lts b/docker/Dockerfile.ci.lts deleted file mode 100644 index 7da27a03f1d..00000000000 --- a/docker/Dockerfile.ci.lts +++ /dev/null @@ -1,98 +0,0 @@ -# syntax=docker/dockerfile:1.3-labs - -ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - -FROM $FROM_IMAGE_NAME as build_mamba -WORKDIR /opt -ARG WHEEL_DIR -RUN MAMBA_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/state-spaces/mamba.git@v2.0.3 -w $WHEEL_DIR - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as build_causalconv1d -WORKDIR /opt -ARG WHEEL_DIR -RUN CAUSAL_CONV1D_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/Dao-AILab/causal-conv1d.git@v1.2.2.post1 -w $WHEEL_DIR - -FROM $FROM_IMAGE_NAME as build_groupedgemm -WORKDIR /opt -ARG WHEEL_DIR -RUN pip3 wheel -v git+https://github.com/fanshiqing/grouped_gemm@v1.1.2 -w $WHEEL_DIR - - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as main -ENV DEBIAN_FRONTEND=noninteractive - -RUN bash -ex <<"EOF" - apt-get update - apt-get install -y --no-install-recommends gettext python3-venv psmisc - apt-get clean - python -m venv /opt/jet - wget https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -O /usr/local/bin/yq - chmod a+x /usr/local/bin/yq -EOF - -ARG UV_VERSION=0.7.2 -ENV PATH="/root/.local/bin:$PATH" -RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh -ENV UV_PROJECT_ENVIRONMENT=/opt/venv -ENV PATH="$UV_PROJECT_ENVIRONMENT/bin:$PATH" -ENV VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT -ENV UV_LINK_MODE=copy - -RUN -ARG WHEEL_DIR -COPY README.md pyproject.toml uv.lock /workspace/ -COPY megatron/core/__init__.py /workspace/megatron/core/ -COPY megatron/core/package_info.py /workspace/megatron/core/ -COPY docker/common/ /workspace/docker/common/ -COPY --from=build_mamba $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_causalconv1d $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_groupedgemm $WHEEL_DIR/*.whl $WHEEL_DIR/ -RUN bash -ex <<"EOF" - uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages - - uv sync --extra lts --extra mlm --link-mode copy --locked \ - --no-install-package torch \ - --no-install-package torchvision \ - --no-install-package triton \ - --no-install-package nvidia-cublas-cu12 \ - --no-install-package nvidia-cuda-cupti-cu12 \ - --no-install-package nvidia-cuda-nvrtc-cu12 \ - --no-install-package nvidia-cuda-runtime-cu12 \ - --no-install-package nvidia-cudnn-cu12 \ - --no-install-package nvidia-cufft-cu12 \ - --no-install-package nvidia-cufile-cu12 \ - --no-install-package nvidia-curand-cu12 \ - --no-install-package nvidia-cusolver-cu12 \ - --no-install-package nvidia-cusparse-cu12 \ - --no-install-package nvidia-cusparselt-cu12 \ - --no-install-package nvidia-nccl-cu12 - - bash docker/common/install_source_wheels.sh --input-wheel-dir $WHEEL_DIR/ --environment lts -EOF -ENV PYTHONPATH="/opt/megatron-lm:$PYTHONPATH" -COPY assets/ /opt/data/ -ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python - -##### For NVIDIANS only ##### -FROM main as jet -ARG JET_API_VERSION -ENV PATH="$PATH:/opt/jet/bin" -RUN --mount=type=secret,id=JET_INDEX_URLS bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - python -m venv /opt/jet - /opt/jet/bin/pip install --no-cache-dir $JET_INDEX_URLS \ - jet-api==$JET_API_VERSION -EOF - -RUN --mount=type=secret,id=JET_INDEX_URLS \ - --mount=type=secret,id=LOGGER_INDEX_URL bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - LOGGER_INDEX_URL=$(cat /run/secrets/LOGGER_INDEX_URL) - uv pip install --no-cache-dir --upgrade $LOGGER_INDEX_URL "one-logger" - uv pip install --no-cache-dir --upgrade "setuptools<80.0.0" - uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=3.0" -EOF -### \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3dcfe98a9b7..9dfa88e541e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ mlm = ["flask-restful", "sentencepiece", "tiktoken", "wandb", "transformers"] dev = [ "nvidia-modelopt[torch]; sys_platform != 'darwin'", "transformer-engine[pytorch]>=2.9.0a0,<2.10.0", - "nvidia-resiliency-ext>=0.4.0a0,<0.5.0", + "nvidia-resiliency-ext", "tqdm", "einops~=0.8", "tensorstore~=0.1,!=0.1.46,!=0.1.72", @@ -89,13 +89,20 @@ dev = [ lts = [ "tqdm", - "einops", - "tensorstore!=0.1.46,!=0.1.72", - "nvtx", - "transformers", - "zarr", + "einops~=0.8", + "tensorstore~=0.1,!=0.1.46,!=0.1.72", + "nvtx~=0.2", + "multi-storage-client~=0.27", + "opentelemetry-api~=1.33.1", "setuptools<80.0.0", + "mamba-ssm~=2.2", + "causal-conv1d~=1.5", + "nv-grouped-gemm~=1.1", + "megatron-energon[av_decode]~=6.0", + "av<16.0.0", # At the time, av 16.0.0 is not compatible with Python 3.12 + "flashinfer-python", "wget", + "onnxscript", ] [dependency-groups] diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json new file mode 100644 index 00000000000..3f7a0c2cac5 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84081, + "2": 10.8354, + "3": 10.83237, + "4": 10.81882, + "5": 10.84098, + "6": 10.87094, + "7": 10.83285, + "8": 10.8395, + "9": 10.84275, + "10": 10.80913, + "11": 10.85185, + "12": 10.84426, + "13": 10.86366, + "14": 10.86332, + "15": 10.80028, + "16": 10.79303, + "17": 10.7753, + "18": 10.80133, + "19": 10.79138, + "20": 10.70502, + "21": 10.68161, + "22": 10.56472, + "23": 10.70185, + "24": 10.58, + "25": 10.5355, + "26": 10.607, + "27": 10.59378, + "28": 10.56083, + "29": 10.57494, + "30": 10.35506, + "31": 10.12664, + "32": 10.46551, + "33": 10.45216, + "34": 10.22453, + "35": 10.27096, + "36": 10.22158, + "37": 10.33994, + "38": 10.18651, + "39": 10.39397, + "40": 10.07811, + "41": 10.13811, + "42": 10.20175, + "43": 9.83808, + "44": 9.94297, + "45": 9.82336, + "46": 9.82165, + "47": 10.13421, + "48": 9.84058, + "49": 9.52119, + "50": 9.90123 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1539.0, + "2": 1717.0, + "3": 1773.0, + "4": 1809.0, + "5": 1932.0, + "6": 1836.0, + "7": 1808.0, + "8": 1638.0, + "9": 1924.0, + "10": 1388.0, + "11": 1978.0, + "12": 1875.0, + "13": 1894.0, + "14": 1832.0, + "15": 1945.0, + "16": 1966.0, + "17": 1779.0, + "18": 1731.0, + "19": 1812.0, + "20": 1744.0, + "21": 1910.0, + "22": 1717.0, + "23": 2079.0, + "24": 1636.0, + "25": 1644.0, + "26": 1812.0, + "27": 1939.0, + "28": 1904.0, + "29": 2001.0, + "30": 2019.0, + "31": 1661.0, + "32": 1904.0, + "33": 2040.0, + "34": 1944.0, + "35": 1955.0, + "36": 1968.0, + "37": 2344.0, + "38": 2300.0, + "39": 2418.0, + "40": 2263.0, + "41": 2357.0, + "42": 2285.0, + "43": 1988.0, + "44": 2123.0, + "45": 2218.0, + "46": 2349.0, + "47": 2594.0, + "48": 2506.0, + "49": 2331.0, + "50": 2374.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4576453120.0, + "2": 4673069568.0, + "3": 4673069568.0, + "4": 4673069568.0, + "5": 4673069568.0, + "6": 4673069568.0, + "7": 4673069568.0, + "8": 4673069568.0, + "9": 4673069568.0, + "10": 4673069568.0, + "11": 4673069568.0, + "12": 4673069568.0, + "13": 4673069568.0, + "14": 4673069568.0, + "15": 4673069568.0, + "16": 4673069568.0, + "17": 4673069568.0, + "18": 4673069568.0, + "19": 4673069568.0, + "20": 4673069568.0, + "21": 4673069568.0, + "22": 4673069568.0, + "23": 4673069568.0, + "24": 4673069568.0, + "25": 4673069568.0, + "26": 4673069568.0, + "27": 4673069568.0, + "28": 4673069568.0, + "29": 4673069568.0, + "30": 4673069568.0, + "31": 4673069568.0, + "32": 4673069568.0, + "33": 4673069568.0, + "34": 4673069568.0, + "35": 4673069568.0, + "36": 4673069568.0, + "37": 4673069568.0, + "38": 4673069568.0, + "39": 4673069568.0, + "40": 4673069568.0, + "41": 4673069568.0, + "42": 4673069568.0, + "43": 4673069568.0, + "44": 4673069568.0, + "45": 4673069568.0, + "46": 4673069568.0, + "47": 4673069568.0, + "48": 4673069568.0, + "49": 4673069568.0, + "50": 4673069568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.18387, + "2": 0.12504, + "3": 0.09705, + "4": 0.09652, + "5": 0.09748, + "6": 0.09683, + "7": 0.09565, + "8": 0.09466, + "9": 0.09428, + "10": 0.09486, + "11": 0.09436, + "12": 0.09386, + "13": 0.09434, + "14": 0.09599, + "15": 0.09464, + "16": 0.0943, + "17": 0.09447, + "18": 0.09424, + "19": 0.0942, + "20": 0.09425, + "21": 0.09401, + "22": 0.09476, + "23": 0.09408, + "24": 0.09462, + "25": 0.09414, + "26": 0.09442, + "27": 0.0939, + "28": 0.09352, + "29": 0.09364, + "30": 0.09376, + "31": 0.09494, + "32": 0.09358, + "33": 0.09378, + "34": 0.09361, + "35": 0.09442, + "36": 0.09437, + "37": 0.09367, + "38": 0.0934, + "39": 0.09328, + "40": 0.09295, + "41": 0.09331, + "42": 0.09302, + "43": 0.09373, + "44": 0.09287, + "45": 0.09264, + "46": 0.10047, + "47": 0.09374, + "48": 0.09248, + "49": 0.09248, + "50": 0.09148 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json index a5a8b2f5f7e..dd30f7144c7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8401, - "5": 10.84032, - "10": 10.8134, - "15": 10.80276, - "20": 10.705, - "25": 10.53849, - "30": 10.35521, - "35": 10.27148, - "40": 10.08049, - "45": 9.8229, - "50": 9.90118 + "1": 10.84081, + "2": 10.8354, + "3": 10.83237, + "4": 10.81882, + "5": 10.84098, + "6": 10.87094, + "7": 10.83285, + "8": 10.8395, + "9": 10.84275, + "10": 10.80913, + "11": 10.85185, + "12": 10.84426, + "13": 10.86366, + "14": 10.86332, + "15": 10.80028, + "16": 10.79303, + "17": 10.7753, + "18": 10.80133, + "19": 10.79138, + "20": 10.70502, + "21": 10.68161, + "22": 10.56472, + "23": 10.70185, + "24": 10.58, + "25": 10.5355, + "26": 10.607, + "27": 10.59378, + "28": 10.56083, + "29": 10.57494, + "30": 10.35506, + "31": 10.12664, + "32": 10.46551, + "33": 10.45216, + "34": 10.22453, + "35": 10.27096, + "36": 10.22158, + "37": 10.33994, + "38": 10.18651, + "39": 10.39397, + "40": 10.07811, + "41": 10.13811, + "42": 10.20175, + "43": 9.83808, + "44": 9.94297, + "45": 9.82336, + "46": 9.82165, + "47": 10.13421, + "48": 9.84058, + "49": 9.52119, + "50": 9.90123 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1670.0, - "5": 1970.0, - "10": 1397.0, - "15": 1936.0, - "20": 1715.0, - "25": 1703.0, - "30": 1957.0, - "35": 1991.0, - "40": 2239.0, - "45": 2144.0, - "50": 2377.0 + "1": 1539.0, + "2": 1717.0, + "3": 1773.0, + "4": 1809.0, + "5": 1932.0, + "6": 1836.0, + "7": 1808.0, + "8": 1638.0, + "9": 1924.0, + "10": 1388.0, + "11": 1978.0, + "12": 1875.0, + "13": 1894.0, + "14": 1832.0, + "15": 1945.0, + "16": 1966.0, + "17": 1779.0, + "18": 1731.0, + "19": 1812.0, + "20": 1744.0, + "21": 1910.0, + "22": 1717.0, + "23": 2079.0, + "24": 1636.0, + "25": 1644.0, + "26": 1812.0, + "27": 1939.0, + "28": 1904.0, + "29": 2001.0, + "30": 2019.0, + "31": 1661.0, + "32": 1904.0, + "33": 2040.0, + "34": 1944.0, + "35": 1955.0, + "36": 1968.0, + "37": 2344.0, + "38": 2300.0, + "39": 2418.0, + "40": 2263.0, + "41": 2357.0, + "42": 2285.0, + "43": 1988.0, + "44": 2123.0, + "45": 2218.0, + "46": 2349.0, + "47": 2594.0, + "48": 2506.0, + "49": 2331.0, + "50": 2374.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 552238592.0, - "5": 552238592.0, - "10": 552238592.0, - "15": 552238592.0, - "20": 552238592.0, - "25": 552238592.0, - "30": 552238592.0, - "35": 552238592.0, - "40": 552238592.0, - "45": 552238592.0, - "50": 552238592.0 + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 4576563200.0, - "5": 4673179648.0, - "10": 4673179648.0, - "15": 4673179648.0, - "20": 4673179648.0, - "25": 4673179648.0, - "30": 4673179648.0, - "35": 4673179648.0, - "40": 4673179648.0, - "45": 4673179648.0, - "50": 4673179648.0 + "1": 4576453120.0, + "2": 4673069568.0, + "3": 4673069568.0, + "4": 4673069568.0, + "5": 4673069568.0, + "6": 4673069568.0, + "7": 4673069568.0, + "8": 4673069568.0, + "9": 4673069568.0, + "10": 4673069568.0, + "11": 4673069568.0, + "12": 4673069568.0, + "13": 4673069568.0, + "14": 4673069568.0, + "15": 4673069568.0, + "16": 4673069568.0, + "17": 4673069568.0, + "18": 4673069568.0, + "19": 4673069568.0, + "20": 4673069568.0, + "21": 4673069568.0, + "22": 4673069568.0, + "23": 4673069568.0, + "24": 4673069568.0, + "25": 4673069568.0, + "26": 4673069568.0, + "27": 4673069568.0, + "28": 4673069568.0, + "29": 4673069568.0, + "30": 4673069568.0, + "31": 4673069568.0, + "32": 4673069568.0, + "33": 4673069568.0, + "34": 4673069568.0, + "35": 4673069568.0, + "36": 4673069568.0, + "37": 4673069568.0, + "38": 4673069568.0, + "39": 4673069568.0, + "40": 4673069568.0, + "41": 4673069568.0, + "42": 4673069568.0, + "43": 4673069568.0, + "44": 4673069568.0, + "45": 4673069568.0, + "46": 4673069568.0, + "47": 4673069568.0, + "48": 4673069568.0, + "49": 4673069568.0, + "50": 4673069568.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.0358, - "5": 0.10098, - "10": 0.10007, - "15": 0.09866, - "20": 0.0986, - "25": 0.09963, - "30": 0.09768, - "35": 0.09735, - "40": 0.09791, - "45": 0.09757, - "50": 0.09769 + "1": 4.57734, + "2": 0.12447, + "3": 0.1105, + "4": 0.11652, + "5": 0.11171, + "6": 0.10268, + "7": 0.0964, + "8": 0.09397, + "9": 0.09475, + "10": 0.09372, + "11": 0.09325, + "12": 0.09309, + "13": 0.09305, + "14": 0.09354, + "15": 0.09324, + "16": 0.09342, + "17": 0.09327, + "18": 0.09347, + "19": 0.09283, + "20": 0.09308, + "21": 0.09266, + "22": 0.09487, + "23": 0.09318, + "24": 0.09338, + "25": 0.09306, + "26": 0.09374, + "27": 0.09386, + "28": 0.09412, + "29": 0.09395, + "30": 0.09393, + "31": 0.09439, + "32": 0.09481, + "33": 0.09338, + "34": 0.09466, + "35": 0.0936, + "36": 0.09463, + "37": 0.09316, + "38": 0.09572, + "39": 0.09295, + "40": 0.09592, + "41": 0.09322, + "42": 0.09468, + "43": 0.09488, + "44": 0.09323, + "45": 0.09265, + "46": 0.09574, + "47": 0.09267, + "48": 0.09592, + "49": 0.09356, + "50": 0.09502 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..d8d087006d9 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84454, + "13": 10.8632, + "14": 10.86353, + "15": 10.79985, + "16": 10.79265, + "17": 10.77473, + "18": 10.80156, + "19": 10.79155, + "20": 10.70512, + "21": 10.68174, + "22": 10.56547, + "23": 10.70145, + "24": 10.5789, + "25": 10.53597, + "26": 10.60745, + "27": 10.59418, + "28": 10.56116, + "29": 10.57573, + "30": 10.35474, + "31": 10.12618, + "32": 10.46569, + "33": 10.45235, + "34": 10.22491, + "35": 10.27088, + "36": 10.22167, + "37": 10.33935, + "38": 10.18639, + "39": 10.39432, + "40": 10.07794, + "41": 10.13875, + "42": 10.20184, + "43": 9.83819, + "44": 9.94273, + "45": 9.82299, + "46": 9.82187, + "47": 10.13444, + "48": 9.84097, + "49": 9.52094, + "50": 9.90107, + "51": 9.83459, + "52": 9.73231, + "53": 10.04881, + "54": 9.93895, + "55": 9.86297, + "56": 9.613, + "57": 9.46964, + "58": 9.81136, + "59": 9.57107, + "60": 9.48153, + "61": 9.67881, + "62": 9.96579, + "63": 9.35276, + "64": 9.75644, + "65": 8.93769, + "66": 9.68152, + "67": 9.35669, + "68": 9.76806, + "69": 9.7739, + "70": 9.71012, + "71": 9.60009, + "72": 9.56796, + "73": 9.47739, + "74": 8.93177, + "75": 9.40721, + "76": 9.06847, + "77": 10.0464, + "78": 9.70984, + "79": 9.35731, + "80": 9.38978, + "81": 9.4662, + "82": 9.68056, + "83": 9.29144, + "84": 9.40194, + "85": 9.59734, + "86": 9.06207, + "87": 9.57921, + "88": 9.73262, + "89": 9.58838, + "90": 9.80354, + "91": 9.31991, + "92": 9.35013, + "93": 9.06378, + "94": 8.81909, + "95": 9.50572, + "96": 9.51068, + "97": 9.29244, + "98": 9.65579, + "99": 8.87401, + "100": 9.38837 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1963.0, + "12": 1833.0, + "13": 1863.0, + "14": 1928.0, + "15": 1948.0, + "16": 1999.0, + "17": 1838.0, + "18": 1703.0, + "19": 1898.0, + "20": 1610.0, + "21": 1944.0, + "22": 1728.0, + "23": 2116.0, + "24": 1613.0, + "25": 1634.0, + "26": 1788.0, + "27": 2059.0, + "28": 2067.0, + "29": 1978.0, + "30": 1929.0, + "31": 1782.0, + "32": 1850.0, + "33": 2169.0, + "34": 1815.0, + "35": 2007.0, + "36": 2010.0, + "37": 2385.0, + "38": 2413.0, + "39": 2474.0, + "40": 2254.0, + "41": 2373.0, + "42": 2253.0, + "43": 1900.0, + "44": 2058.0, + "45": 2153.0, + "46": 2385.0, + "47": 2514.0, + "48": 2475.0, + "49": 2362.0, + "50": 2335.0, + "51": 2452.0, + "52": 2576.0, + "53": 2914.0, + "54": 2741.0, + "55": 2408.0, + "56": 2650.0, + "57": 2264.0, + "58": 2853.0, + "59": 2757.0, + "60": 2509.0, + "61": 3076.0, + "62": 2709.0, + "63": 2563.0, + "64": 3041.0, + "65": 2687.0, + "66": 3089.0, + "67": 2767.0, + "68": 2930.0, + "69": 2911.0, + "70": 3286.0, + "71": 3105.0, + "72": 2507.0, + "73": 3063.0, + "74": 2022.0, + "75": 2763.0, + "76": 3002.0, + "77": 3382.0, + "78": 3470.0, + "79": 3109.0, + "80": 3357.0, + "81": 3798.0, + "82": 3348.0, + "83": 2763.0, + "84": 3271.0, + "85": 3245.0, + "86": 2587.0, + "87": 3650.0, + "88": 3103.0, + "89": 3471.0, + "90": 3086.0, + "91": 3050.0, + "92": 3368.0, + "93": 2828.0, + "94": 3495.0, + "95": 3424.0, + "96": 3559.0, + "97": 3289.0, + "98": 3727.0, + "99": 3275.0, + "100": 3401.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0, + "51": 552128512.0, + "52": 552128512.0, + "53": 552128512.0, + "54": 552128512.0, + "55": 552128512.0, + "56": 552128512.0, + "57": 552128512.0, + "58": 552128512.0, + "59": 552128512.0, + "60": 552128512.0, + "61": 552128512.0, + "62": 552128512.0, + "63": 552128512.0, + "64": 552128512.0, + "65": 552128512.0, + "66": 552128512.0, + "67": 552128512.0, + "68": 552128512.0, + "69": 552128512.0, + "70": 552128512.0, + "71": 552128512.0, + "72": 552128512.0, + "73": 552128512.0, + "74": 552128512.0, + "75": 552128512.0, + "76": 552128512.0, + "77": 552128512.0, + "78": 552128512.0, + "79": 552128512.0, + "80": 552128512.0, + "81": 552128512.0, + "82": 552128512.0, + "83": 552128512.0, + "84": 552128512.0, + "85": 552128512.0, + "86": 552128512.0, + "87": 552128512.0, + "88": 552128512.0, + "89": 552128512.0, + "90": 552128512.0, + "91": 552128512.0, + "92": 552128512.0, + "93": 552128512.0, + "94": 552128512.0, + "95": 552128512.0, + "96": 552128512.0, + "97": 552128512.0, + "98": 552128512.0, + "99": 552128512.0, + "100": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2615097856.0, + "2": 2711714304.0, + "3": 2711714304.0, + "4": 2711714304.0, + "5": 2711714304.0, + "6": 2711714304.0, + "7": 2711714304.0, + "8": 2711714304.0, + "9": 2711714304.0, + "10": 2711714304.0, + "11": 2711714304.0, + "12": 2711714304.0, + "13": 2711714304.0, + "14": 2711714304.0, + "15": 2711714304.0, + "16": 2711714304.0, + "17": 2711714304.0, + "18": 2711714304.0, + "19": 2711714304.0, + "20": 2711714304.0, + "21": 2711714304.0, + "22": 2711714304.0, + "23": 2711714304.0, + "24": 2711714304.0, + "25": 2711714304.0, + "26": 2711714304.0, + "27": 2711714304.0, + "28": 2711714304.0, + "29": 2711714304.0, + "30": 2711714304.0, + "31": 2711714304.0, + "32": 2711714304.0, + "33": 2711714304.0, + "34": 2711714304.0, + "35": 2711714304.0, + "36": 2711714304.0, + "37": 2711714304.0, + "38": 2711714304.0, + "39": 2711714304.0, + "40": 2711714304.0, + "41": 2711714304.0, + "42": 2711714304.0, + "43": 2711714304.0, + "44": 2711714304.0, + "45": 2711714304.0, + "46": 2711714304.0, + "47": 2711714304.0, + "48": 2711714304.0, + "49": 2711714304.0, + "50": 2711714304.0, + "51": 2711714304.0, + "52": 2711714304.0, + "53": 2711714304.0, + "54": 2711714304.0, + "55": 2711714304.0, + "56": 2711714304.0, + "57": 2711714304.0, + "58": 2711714304.0, + "59": 2711714304.0, + "60": 2711714304.0, + "61": 2711714304.0, + "62": 2711714304.0, + "63": 2711714304.0, + "64": 2711714304.0, + "65": 2711714304.0, + "66": 2711714304.0, + "67": 2711714304.0, + "68": 2711714304.0, + "69": 2711714304.0, + "70": 2711714304.0, + "71": 2711714304.0, + "72": 2711714304.0, + "73": 2711714304.0, + "74": 2711714304.0, + "75": 2711714304.0, + "76": 2711714304.0, + "77": 2711714304.0, + "78": 2711714304.0, + "79": 2711714304.0, + "80": 2711714304.0, + "81": 2711714304.0, + "82": 2711714304.0, + "83": 2711714304.0, + "84": 2711714304.0, + "85": 2711714304.0, + "86": 2711714304.0, + "87": 2711714304.0, + "88": 2711714304.0, + "89": 2711714304.0, + "90": 2711714304.0, + "91": 2711714304.0, + "92": 2711714304.0, + "93": 2711714304.0, + "94": 2711714304.0, + "95": 2711714304.0, + "96": 2711714304.0, + "97": 2711714304.0, + "98": 2711714304.0, + "99": 2711714304.0, + "100": 2711714304.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.22026, + "2": 0.11508, + "3": 0.08784, + "4": 0.08853, + "5": 0.08761, + "6": 0.08872, + "7": 0.08738, + "8": 0.08924, + "9": 0.08542, + "10": 0.08527, + "11": 0.08496, + "12": 0.08437, + "13": 0.08471, + "14": 0.08501, + "15": 0.08468, + "16": 0.08444, + "17": 0.08421, + "18": 0.08402, + "19": 0.08444, + "20": 0.0841, + "21": 0.08395, + "22": 0.08442, + "23": 0.0846, + "24": 0.08394, + "25": 0.08371, + "26": 0.08416, + "27": 0.08354, + "28": 0.08445, + "29": 0.08405, + "30": 0.08414, + "31": 0.08444, + "32": 0.08369, + "33": 0.08356, + "34": 0.08435, + "35": 0.08405, + "36": 0.08358, + "37": 0.08349, + "38": 0.08439, + "39": 0.0837, + "40": 0.08444, + "41": 0.08399, + "42": 0.0835, + "43": 0.0843, + "44": 0.08389, + "45": 0.084, + "46": 0.08426, + "47": 0.0842, + "48": 0.08434, + "49": 0.08385, + "50": 0.08424, + "51": 0.09846, + "52": 0.08909, + "53": 0.08511, + "54": 0.0849, + "55": 0.1206, + "56": 0.0845, + "57": 0.08446, + "58": 0.08474, + "59": 0.08505, + "60": 0.08422, + "61": 0.08413, + "62": 0.0845, + "63": 0.08441, + "64": 0.08486, + "65": 0.08527, + "66": 0.08442, + "67": 0.08533, + "68": 0.08468, + "69": 0.08469, + "70": 0.08503, + "71": 0.08424, + "72": 0.085, + "73": 0.08469, + "74": 0.08484, + "75": 0.08396, + "76": 0.08437, + "77": 0.08458, + "78": 0.08553, + "79": 0.08492, + "80": 0.08459, + "81": 0.08431, + "82": 0.08515, + "83": 0.08407, + "84": 0.08429, + "85": 0.08413, + "86": 0.08499, + "87": 0.08442, + "88": 0.08461, + "89": 0.08478, + "90": 0.08469, + "91": 0.08469, + "92": 0.08478, + "93": 0.08453, + "94": 0.0842, + "95": 0.08391, + "96": 0.08383, + "97": 0.08459, + "98": 0.08469, + "99": 0.085, + "100": 0.08518 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json index 66e69ad5f5e..9dad9972e22 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8401, - "5": 10.84032, - "10": 10.8134, - "15": 10.80276, - "20": 10.705, - "25": 10.53849, - "30": 10.35521, - "35": 10.27148, - "40": 10.08049, - "45": 9.8229, - "50": 9.90118, - "55": 9.86424, - "60": 9.4803, - "65": 8.93743, - "70": 9.71026, - "75": 9.4088, - "80": 9.39078, - "85": 9.59741, - "90": 9.80389, - "95": 9.50562, - "100": 9.38808 + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84454, + "13": 10.8632, + "14": 10.86353, + "15": 10.79985, + "16": 10.79265, + "17": 10.77473, + "18": 10.80156, + "19": 10.79155, + "20": 10.70512, + "21": 10.68174, + "22": 10.56547, + "23": 10.70145, + "24": 10.5789, + "25": 10.53597, + "26": 10.60745, + "27": 10.59418, + "28": 10.56116, + "29": 10.57573, + "30": 10.35474, + "31": 10.12618, + "32": 10.46569, + "33": 10.45235, + "34": 10.22491, + "35": 10.27088, + "36": 10.22167, + "37": 10.33935, + "38": 10.18639, + "39": 10.39432, + "40": 10.07794, + "41": 10.13875, + "42": 10.20184, + "43": 9.83819, + "44": 9.94273, + "45": 9.82299, + "46": 9.82187, + "47": 10.13444, + "48": 9.84097, + "49": 9.52094, + "50": 9.90107, + "51": 9.83459, + "52": 9.73231, + "53": 10.04881, + "54": 9.93895, + "55": 9.86297, + "56": 9.613, + "57": 9.46964, + "58": 9.81136, + "59": 9.57107, + "60": 9.48153, + "61": 9.67881, + "62": 9.96579, + "63": 9.35276, + "64": 9.75644, + "65": 8.93769, + "66": 9.68152, + "67": 9.35669, + "68": 9.76806, + "69": 9.7739, + "70": 9.71012, + "71": 9.60009, + "72": 9.56796, + "73": 9.47739, + "74": 8.93177, + "75": 9.40721, + "76": 9.06847, + "77": 10.0464, + "78": 9.70984, + "79": 9.35731, + "80": 9.38978, + "81": 9.4662, + "82": 9.68056, + "83": 9.29144, + "84": 9.40194, + "85": 9.59734, + "86": 9.06207, + "87": 9.57921, + "88": 9.73262, + "89": 9.58838, + "90": 9.80354, + "91": 9.31991, + "92": 9.35013, + "93": 9.06378, + "94": 8.81909, + "95": 9.50572, + "96": 9.51068, + "97": 9.29244, + "98": 9.65579, + "99": 8.87401, + "100": 9.38837 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1670.0, - "5": 1970.0, - "10": 1397.0, - "15": 1936.0, - "20": 1715.0, - "25": 1703.0, - "30": 1957.0, - "35": 1991.0, - "40": 2239.0, - "45": 2144.0, - "50": 2377.0, - "55": 2436.0, - "60": 2445.0, - "65": 2645.0, - "70": 3337.0, - "75": 2726.0, - "80": 3356.0, - "85": 3336.0, - "90": 3044.0, - "95": 3484.0, - "100": 3467.0 + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1963.0, + "12": 1833.0, + "13": 1863.0, + "14": 1928.0, + "15": 1948.0, + "16": 1999.0, + "17": 1838.0, + "18": 1703.0, + "19": 1898.0, + "20": 1610.0, + "21": 1944.0, + "22": 1728.0, + "23": 2116.0, + "24": 1613.0, + "25": 1634.0, + "26": 1788.0, + "27": 2059.0, + "28": 2067.0, + "29": 1978.0, + "30": 1929.0, + "31": 1782.0, + "32": 1850.0, + "33": 2169.0, + "34": 1815.0, + "35": 2007.0, + "36": 2010.0, + "37": 2385.0, + "38": 2413.0, + "39": 2474.0, + "40": 2254.0, + "41": 2373.0, + "42": 2253.0, + "43": 1900.0, + "44": 2058.0, + "45": 2153.0, + "46": 2385.0, + "47": 2514.0, + "48": 2475.0, + "49": 2362.0, + "50": 2335.0, + "51": 2452.0, + "52": 2576.0, + "53": 2914.0, + "54": 2741.0, + "55": 2408.0, + "56": 2650.0, + "57": 2264.0, + "58": 2853.0, + "59": 2757.0, + "60": 2509.0, + "61": 3076.0, + "62": 2709.0, + "63": 2563.0, + "64": 3041.0, + "65": 2687.0, + "66": 3089.0, + "67": 2767.0, + "68": 2930.0, + "69": 2911.0, + "70": 3286.0, + "71": 3105.0, + "72": 2507.0, + "73": 3063.0, + "74": 2022.0, + "75": 2763.0, + "76": 3002.0, + "77": 3382.0, + "78": 3470.0, + "79": 3109.0, + "80": 3357.0, + "81": 3798.0, + "82": 3348.0, + "83": 2763.0, + "84": 3271.0, + "85": 3245.0, + "86": 2587.0, + "87": 3650.0, + "88": 3103.0, + "89": 3471.0, + "90": 3086.0, + "91": 3050.0, + "92": 3368.0, + "93": 2828.0, + "94": 3495.0, + "95": 3424.0, + "96": 3559.0, + "97": 3289.0, + "98": 3727.0, + "99": 3275.0, + "100": 3401.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 552238592.0, - "5": 552238592.0, - "10": 552238592.0, - "15": 552238592.0, - "20": 552238592.0, - "25": 552238592.0, - "30": 552238592.0, - "35": 552238592.0, - "40": 552238592.0, - "45": 552238592.0, - "50": 552238592.0, - "55": 552238592.0, - "60": 552238592.0, - "65": 552238592.0, - "70": 552238592.0, - "75": 552238592.0, - "80": 552238592.0, - "85": 552238592.0, - "90": 552238592.0, - "95": 552238592.0, - "100": 552238592.0 + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0, + "51": 552128512.0, + "52": 552128512.0, + "53": 552128512.0, + "54": 552128512.0, + "55": 552128512.0, + "56": 552128512.0, + "57": 552128512.0, + "58": 552128512.0, + "59": 552128512.0, + "60": 552128512.0, + "61": 552128512.0, + "62": 552128512.0, + "63": 552128512.0, + "64": 552128512.0, + "65": 552128512.0, + "66": 552128512.0, + "67": 552128512.0, + "68": 552128512.0, + "69": 552128512.0, + "70": 552128512.0, + "71": 552128512.0, + "72": 552128512.0, + "73": 552128512.0, + "74": 552128512.0, + "75": 552128512.0, + "76": 552128512.0, + "77": 552128512.0, + "78": 552128512.0, + "79": 552128512.0, + "80": 552128512.0, + "81": 552128512.0, + "82": 552128512.0, + "83": 552128512.0, + "84": 552128512.0, + "85": 552128512.0, + "86": 552128512.0, + "87": 552128512.0, + "88": 552128512.0, + "89": 552128512.0, + "90": 552128512.0, + "91": 552128512.0, + "92": 552128512.0, + "93": 552128512.0, + "94": 552128512.0, + "95": 552128512.0, + "96": 552128512.0, + "97": 552128512.0, + "98": 552128512.0, + "99": 552128512.0, + "100": 552128512.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 4576563200.0, - "5": 4673179648.0, - "10": 4673179648.0, - "15": 4673179648.0, - "20": 4673179648.0, - "25": 4673179648.0, - "30": 4673179648.0, - "35": 4673179648.0, - "40": 4673179648.0, - "45": 4673179648.0, - "50": 4673179648.0, - "55": 4673179648.0, - "60": 4673179648.0, - "65": 4673179648.0, - "70": 4673179648.0, - "75": 4673179648.0, - "80": 4673179648.0, - "85": 4673179648.0, - "90": 4673179648.0, - "95": 4673179648.0, - "100": 4673179648.0 + "1": 2615097856.0, + "2": 2711714304.0, + "3": 2711714304.0, + "4": 2711714304.0, + "5": 2711714304.0, + "6": 2711714304.0, + "7": 2711714304.0, + "8": 2711714304.0, + "9": 2711714304.0, + "10": 2711714304.0, + "11": 2711714304.0, + "12": 2711714304.0, + "13": 2711714304.0, + "14": 2711714304.0, + "15": 2711714304.0, + "16": 2711714304.0, + "17": 2711714304.0, + "18": 2711714304.0, + "19": 2711714304.0, + "20": 2711714304.0, + "21": 2711714304.0, + "22": 2711714304.0, + "23": 2711714304.0, + "24": 2711714304.0, + "25": 2711714304.0, + "26": 2711714304.0, + "27": 2711714304.0, + "28": 2711714304.0, + "29": 2711714304.0, + "30": 2711714304.0, + "31": 2711714304.0, + "32": 2711714304.0, + "33": 2711714304.0, + "34": 2711714304.0, + "35": 2711714304.0, + "36": 2711714304.0, + "37": 2711714304.0, + "38": 2711714304.0, + "39": 2711714304.0, + "40": 2711714304.0, + "41": 2711714304.0, + "42": 2711714304.0, + "43": 2711714304.0, + "44": 2711714304.0, + "45": 2711714304.0, + "46": 2711714304.0, + "47": 2711714304.0, + "48": 2711714304.0, + "49": 2711714304.0, + "50": 2711714304.0, + "51": 2711714304.0, + "52": 2711714304.0, + "53": 2711714304.0, + "54": 2711714304.0, + "55": 2711714304.0, + "56": 2711714304.0, + "57": 2711714304.0, + "58": 2711714304.0, + "59": 2711714304.0, + "60": 2711714304.0, + "61": 2711714304.0, + "62": 2711714304.0, + "63": 2711714304.0, + "64": 2711714304.0, + "65": 2711714304.0, + "66": 2711714304.0, + "67": 2711714304.0, + "68": 2711714304.0, + "69": 2711714304.0, + "70": 2711714304.0, + "71": 2711714304.0, + "72": 2711714304.0, + "73": 2711714304.0, + "74": 2711714304.0, + "75": 2711714304.0, + "76": 2711714304.0, + "77": 2711714304.0, + "78": 2711714304.0, + "79": 2711714304.0, + "80": 2711714304.0, + "81": 2711714304.0, + "82": 2711714304.0, + "83": 2711714304.0, + "84": 2711714304.0, + "85": 2711714304.0, + "86": 2711714304.0, + "87": 2711714304.0, + "88": 2711714304.0, + "89": 2711714304.0, + "90": 2711714304.0, + "91": 2711714304.0, + "92": 2711714304.0, + "93": 2711714304.0, + "94": 2711714304.0, + "95": 2711714304.0, + "96": 2711714304.0, + "97": 2711714304.0, + "98": 2711714304.0, + "99": 2711714304.0, + "100": 2711714304.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.31173, - "5": 0.10511, - "10": 0.10337, - "15": 0.10143, - "20": 0.10201, - "25": 0.10088, - "30": 0.10154, - "35": 0.1006, - "40": 0.10106, - "45": 0.10038, - "50": 0.10112, - "55": 0.10263, - "60": 0.0993, - "65": 0.09963, - "70": 0.09998, - "75": 0.09811, - "80": 0.09929, - "85": 0.09785, - "90": 0.09909, - "95": 0.09864, - "100": 0.09877 + "1": 4.10688, + "2": 0.11397, + "3": 0.08797, + "4": 0.08663, + "5": 0.08687, + "6": 0.08702, + "7": 0.08653, + "8": 0.08674, + "9": 0.08696, + "10": 0.08678, + "11": 0.08635, + "12": 0.08637, + "13": 0.08738, + "14": 0.08674, + "15": 0.08706, + "16": 0.08684, + "17": 0.08681, + "18": 0.08601, + "19": 0.08591, + "20": 0.08645, + "21": 0.08634, + "22": 0.08598, + "23": 0.08618, + "24": 0.08622, + "25": 0.08632, + "26": 0.08621, + "27": 0.08644, + "28": 0.08581, + "29": 0.08622, + "30": 0.08652, + "31": 0.08679, + "32": 0.08526, + "33": 0.08525, + "34": 0.08525, + "35": 0.08519, + "36": 0.08535, + "37": 0.08568, + "38": 0.0852, + "39": 0.08521, + "40": 0.08523, + "41": 0.08535, + "42": 0.08486, + "43": 0.08614, + "44": 0.08491, + "45": 0.08554, + "46": 0.08508, + "47": 0.08524, + "48": 0.08608, + "49": 0.08565, + "50": 0.08559, + "51": 0.10342, + "52": 0.09048, + "53": 0.08707, + "54": 0.08719, + "55": 0.08631, + "56": 0.11667, + "57": 0.08592, + "58": 0.08517, + "59": 0.08612, + "60": 0.08514, + "61": 0.0855, + "62": 0.08527, + "63": 0.08586, + "64": 0.08556, + "65": 0.08633, + "66": 0.08532, + "67": 0.08593, + "68": 0.08563, + "69": 0.08537, + "70": 0.08538, + "71": 0.08507, + "72": 0.08593, + "73": 0.08623, + "74": 0.08561, + "75": 0.08536, + "76": 0.08551, + "77": 0.08526, + "78": 0.0859, + "79": 0.08518, + "80": 0.08601, + "81": 0.08574, + "82": 0.08618, + "83": 0.08532, + "84": 0.08505, + "85": 0.08545, + "86": 0.08554, + "87": 0.08542, + "88": 0.08575, + "89": 0.0861, + "90": 0.08516, + "91": 0.08552, + "92": 0.08581, + "93": 0.08558, + "94": 0.08577, + "95": 0.08708, + "96": 0.08574, + "97": 0.08543, + "98": 0.0855, + "99": 0.08537, + "100": 0.08541 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json new file mode 100644 index 00000000000..c6d7c9c90fa --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84449, + "13": 10.86322, + "14": 10.86353, + "15": 10.79981, + "16": 10.79262, + "17": 10.77477, + "18": 10.80157, + "19": 10.79148, + "20": 10.70508, + "21": 10.68176, + "22": 10.56548, + "23": 10.70147, + "24": 10.57889, + "25": 10.53597, + "26": 10.60742, + "27": 10.59423, + "28": 10.56119, + "29": 10.57569, + "30": 10.35474, + "31": 10.12616, + "32": 10.46566, + "33": 10.45233, + "34": 10.22493, + "35": 10.27091, + "36": 10.22168, + "37": 10.33936, + "38": 10.18641, + "39": 10.39431, + "40": 10.07792, + "41": 10.13872, + "42": 10.20182, + "43": 9.83818, + "44": 9.94274, + "45": 9.82303, + "46": 9.82185, + "47": 10.13443, + "48": 9.84098, + "49": 9.52095, + "50": 9.90109, + "51": 9.83457, + "52": 9.73232, + "53": 10.0488, + "54": 9.93895, + "55": 9.863, + "56": 9.613, + "57": 9.46966, + "58": 9.81135, + "59": 9.57107, + "60": 9.48155, + "61": 9.6788, + "62": 9.96581, + "63": 9.35273, + "64": 9.75648, + "65": 8.93771, + "66": 9.68153, + "67": 9.35671, + "68": 9.76807, + "69": 9.7739, + "70": 9.71016, + "71": 9.60009, + "72": 9.56793, + "73": 9.4774, + "74": 8.93177, + "75": 9.4072, + "76": 9.06849, + "77": 10.0464, + "78": 9.70988, + "79": 9.35733, + "80": 9.38975, + "81": 9.4662, + "82": 9.68058, + "83": 9.2914, + "84": 9.40191, + "85": 9.59735, + "86": 9.06209, + "87": 9.57922, + "88": 9.73259, + "89": 9.58836, + "90": 9.80354, + "91": 9.31991, + "92": 9.35011, + "93": 9.06382, + "94": 8.81909, + "95": 9.50568, + "96": 9.51071, + "97": 9.29241, + "98": 9.65578, + "99": 8.87401, + "100": 9.38833 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1966.0, + "12": 1767.0, + "13": 1876.0, + "14": 1881.0, + "15": 1928.0, + "16": 1875.0, + "17": 1790.0, + "18": 1736.0, + "19": 1804.0, + "20": 1690.0, + "21": 2008.0, + "22": 1765.0, + "23": 2073.0, + "24": 1618.0, + "25": 1720.0, + "26": 1807.0, + "27": 1861.0, + "28": 2026.0, + "29": 1982.0, + "30": 1981.0, + "31": 1688.0, + "32": 1913.0, + "33": 2123.0, + "34": 1893.0, + "35": 2007.0, + "36": 1987.0, + "37": 2334.0, + "38": 2223.0, + "39": 2417.0, + "40": 2370.0, + "41": 2352.0, + "42": 2269.0, + "43": 1967.0, + "44": 2183.0, + "45": 2150.0, + "46": 2350.0, + "47": 2555.0, + "48": 2463.0, + "49": 2326.0, + "50": 2270.0, + "51": 2508.0, + "52": 2495.0, + "53": 2856.0, + "54": 2692.0, + "55": 2482.0, + "56": 2614.0, + "57": 2283.0, + "58": 2894.0, + "59": 2659.0, + "60": 2561.0, + "61": 3006.0, + "62": 2671.0, + "63": 2488.0, + "64": 3092.0, + "65": 2622.0, + "66": 3108.0, + "67": 2741.0, + "68": 2942.0, + "69": 2983.0, + "70": 3347.0, + "71": 3034.0, + "72": 2438.0, + "73": 3075.0, + "74": 1931.0, + "75": 2722.0, + "76": 2960.0, + "77": 3387.0, + "78": 3268.0, + "79": 3079.0, + "80": 3404.0, + "81": 3674.0, + "82": 3192.0, + "83": 2791.0, + "84": 3224.0, + "85": 3237.0, + "86": 2646.0, + "87": 3840.0, + "88": 3114.0, + "89": 3410.0, + "90": 3184.0, + "91": 3073.0, + "92": 3396.0, + "93": 2711.0, + "94": 3530.0, + "95": 3387.0, + "96": 3530.0, + "97": 3277.0, + "98": 3775.0, + "99": 3421.0, + "100": 3350.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1234585088.0, + "2": 1234585088.0, + "3": 1234585088.0, + "4": 1234585088.0, + "5": 1234585088.0, + "6": 1234585088.0, + "7": 1234585088.0, + "8": 1234585088.0, + "9": 1234585088.0, + "10": 1234585088.0, + "11": 1234585088.0, + "12": 1234585088.0, + "13": 1234585088.0, + "14": 1234585088.0, + "15": 1234585088.0, + "16": 1234585088.0, + "17": 1234585088.0, + "18": 1234585088.0, + "19": 1234585088.0, + "20": 1234585088.0, + "21": 1234585088.0, + "22": 1234585088.0, + "23": 1234585088.0, + "24": 1234585088.0, + "25": 1234585088.0, + "26": 1234585088.0, + "27": 1234585088.0, + "28": 1234585088.0, + "29": 1234585088.0, + "30": 1234585088.0, + "31": 1234585088.0, + "32": 1234585088.0, + "33": 1234585088.0, + "34": 1234585088.0, + "35": 1234585088.0, + "36": 1234585088.0, + "37": 1234585088.0, + "38": 1234585088.0, + "39": 1234585088.0, + "40": 1234585088.0, + "41": 1234585088.0, + "42": 1234585088.0, + "43": 1234585088.0, + "44": 1234585088.0, + "45": 1234585088.0, + "46": 1234585088.0, + "47": 1234585088.0, + "48": 1234585088.0, + "49": 1234585088.0, + "50": 1234585088.0, + "51": 1234585088.0, + "52": 1234585088.0, + "53": 1234585088.0, + "54": 1234585088.0, + "55": 1234585088.0, + "56": 1234585088.0, + "57": 1234585088.0, + "58": 1234585088.0, + "59": 1234585088.0, + "60": 1234585088.0, + "61": 1234585088.0, + "62": 1234585088.0, + "63": 1234585088.0, + "64": 1234585088.0, + "65": 1234585088.0, + "66": 1234585088.0, + "67": 1234585088.0, + "68": 1234585088.0, + "69": 1234585088.0, + "70": 1234585088.0, + "71": 1234585088.0, + "72": 1234585088.0, + "73": 1234585088.0, + "74": 1234585088.0, + "75": 1234585088.0, + "76": 1234585088.0, + "77": 1234585088.0, + "78": 1234585088.0, + "79": 1234585088.0, + "80": 1234585088.0, + "81": 1234585088.0, + "82": 1234585088.0, + "83": 1234585088.0, + "84": 1234585088.0, + "85": 1234585088.0, + "86": 1234585088.0, + "87": 1234585088.0, + "88": 1234585088.0, + "89": 1234585088.0, + "90": 1234585088.0, + "91": 1234585088.0, + "92": 1234585088.0, + "93": 1234585088.0, + "94": 1234585088.0, + "95": 1234585088.0, + "96": 1234585088.0, + "97": 1234585088.0, + "98": 1234585088.0, + "99": 1234585088.0, + "100": 1234585088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1984492544.0, + "2": 2536167424.0, + "3": 2536167424.0, + "4": 2536167424.0, + "5": 2536167424.0, + "6": 2536167424.0, + "7": 2536167424.0, + "8": 2536167424.0, + "9": 2536167424.0, + "10": 2536167424.0, + "11": 2536167424.0, + "12": 2536167424.0, + "13": 2536167424.0, + "14": 2536167424.0, + "15": 2536167424.0, + "16": 2536167424.0, + "17": 2536167424.0, + "18": 2536167424.0, + "19": 2536167424.0, + "20": 2536167424.0, + "21": 2536167424.0, + "22": 2536167424.0, + "23": 2536167424.0, + "24": 2536167424.0, + "25": 2536167424.0, + "26": 2536167424.0, + "27": 2536167424.0, + "28": 2536167424.0, + "29": 2536167424.0, + "30": 2536167424.0, + "31": 2536167424.0, + "32": 2536167424.0, + "33": 2536167424.0, + "34": 2536167424.0, + "35": 2536167424.0, + "36": 2536167424.0, + "37": 2536167424.0, + "38": 2536167424.0, + "39": 2536167424.0, + "40": 2536167424.0, + "41": 2536167424.0, + "42": 2536167424.0, + "43": 2536167424.0, + "44": 2536167424.0, + "45": 2536167424.0, + "46": 2536167424.0, + "47": 2536167424.0, + "48": 2536167424.0, + "49": 2536167424.0, + "50": 2536167424.0, + "51": 2536167424.0, + "52": 2536167424.0, + "53": 2536167424.0, + "54": 2536167424.0, + "55": 2536167424.0, + "56": 2536167424.0, + "57": 2536167424.0, + "58": 2536167424.0, + "59": 2536167424.0, + "60": 2536167424.0, + "61": 2536167424.0, + "62": 2536167424.0, + "63": 2536167424.0, + "64": 2536167424.0, + "65": 2536167424.0, + "66": 2536167424.0, + "67": 2536167424.0, + "68": 2536167424.0, + "69": 2536167424.0, + "70": 2536167424.0, + "71": 2536167424.0, + "72": 2536167424.0, + "73": 2536167424.0, + "74": 2536167424.0, + "75": 2536167424.0, + "76": 2536167424.0, + "77": 2536167424.0, + "78": 2536167424.0, + "79": 2536167424.0, + "80": 2536167424.0, + "81": 2536167424.0, + "82": 2536167424.0, + "83": 2536167424.0, + "84": 2536167424.0, + "85": 2536167424.0, + "86": 2536167424.0, + "87": 2536167424.0, + "88": 2536167424.0, + "89": 2536167424.0, + "90": 2536167424.0, + "91": 2536167424.0, + "92": 2536167424.0, + "93": 2536167424.0, + "94": 2536167424.0, + "95": 2536167424.0, + "96": 2536167424.0, + "97": 2536167424.0, + "98": 2536167424.0, + "99": 2536167424.0, + "100": 2536167424.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.55988, + "2": 0.14641, + "3": 0.12458, + "4": 0.12359, + "5": 0.12323, + "6": 0.12296, + "7": 0.12273, + "8": 0.12322, + "9": 0.12253, + "10": 0.12257, + "11": 0.12186, + "12": 0.12242, + "13": 0.12302, + "14": 0.12233, + "15": 0.12201, + "16": 0.12138, + "17": 0.12155, + "18": 0.12113, + "19": 0.12159, + "20": 0.12132, + "21": 0.12202, + "22": 0.12206, + "23": 0.12186, + "24": 0.12276, + "25": 0.12238, + "26": 0.122, + "27": 0.12177, + "28": 0.12203, + "29": 0.12255, + "30": 0.12285, + "31": 0.1224, + "32": 0.12276, + "33": 0.12359, + "34": 0.12194, + "35": 0.12249, + "36": 0.12276, + "37": 0.12249, + "38": 0.12249, + "39": 0.12333, + "40": 0.12327, + "41": 0.12316, + "42": 0.12307, + "43": 0.12249, + "44": 0.12267, + "45": 0.12282, + "46": 0.12405, + "47": 0.12264, + "48": 0.12412, + "49": 0.12277, + "50": 0.12365, + "51": 0.1271, + "52": 0.12708, + "53": 0.12522, + "54": 0.1263, + "55": 0.12587, + "56": 0.12762, + "57": 0.12527, + "58": 0.12651, + "59": 0.12671, + "60": 0.12654, + "61": 0.12604, + "62": 0.12577, + "63": 0.12494, + "64": 0.12609, + "65": 0.12576, + "66": 0.12652, + "67": 0.12628, + "68": 0.12655, + "69": 0.12565, + "70": 0.12576, + "71": 0.12521, + "72": 0.12593, + "73": 0.12578, + "74": 0.12645, + "75": 0.12537, + "76": 0.12616, + "77": 0.12525, + "78": 0.12803, + "79": 0.1252, + "80": 0.12678, + "81": 0.12525, + "82": 0.12597, + "83": 0.12596, + "84": 0.12603, + "85": 0.1257, + "86": 0.12623, + "87": 0.12511, + "88": 0.12609, + "89": 0.12568, + "90": 0.12585, + "91": 0.12495, + "92": 0.12654, + "93": 0.12549, + "94": 0.12609, + "95": 0.12518, + "96": 0.12593, + "97": 0.12598, + "98": 0.12611, + "99": 0.12441, + "100": 0.12715 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json index f24760a1ccb..bd7ca46935f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.8401, "5": 10.84032, "10": 10.81341, "15": 10.80278, "20": 10.70496, "25": 10.53846, "30": 10.35517, "35": 10.27147, "40": 10.08044, "45": 9.82292, "50": 9.90113, "55": 9.86422, "60": 9.48029, "65": 8.93749, "70": 9.71025, "75": 9.40879, "80": 9.39077, "85": 9.59743, "90": 9.80386, "95": 9.50565, "100": 9.38812}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1670.0, "5": 1970.0, "10": 1436.0, "15": 1918.0, "20": 1786.0, "25": 1610.0, "30": 2039.0, "35": 2001.0, "40": 2321.0, "45": 2205.0, "50": 2365.0, "55": 2489.0, "60": 2508.0, "65": 2719.0, "70": 3241.0, "75": 2643.0, "80": 3368.0, "85": 3336.0, "90": 2961.0, "95": 3533.0, "100": 3432.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1230390272.0, "5": 1230390272.0, "10": 1230390272.0, "15": 1230390272.0, "20": 1230390272.0, "25": 1230390272.0, "30": 1230390272.0, "35": 1230390272.0, "40": 1230390272.0, "45": 1230390272.0, "50": 1230390272.0, "55": 1230390272.0, "60": 1230390272.0, "65": 1230390272.0, "70": 1230390272.0, "75": 1230390272.0, "80": 1230390272.0, "85": 1230390272.0, "90": 1230390272.0, "95": 1230390272.0, "100": 1230390272.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1984492032.0, "5": 2531972608.0, "10": 2531972608.0, "15": 2531972608.0, "20": 2531972608.0, "25": 2531972608.0, "30": 2531972608.0, "35": 2531972608.0, "40": 2531972608.0, "45": 2531972608.0, "50": 2531972608.0, "55": 2531972608.0, "60": 2531972608.0, "65": 2531972608.0, "70": 2531972608.0, "75": 2531972608.0, "80": 2531972608.0, "85": 2531972608.0, "90": 2531972608.0, "95": 2531972608.0, "100": 2531972608.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 6.64146, "5": 0.13017, "10": 0.12892, "15": 0.12866, "20": 0.12752, "25": 0.12949, "30": 0.12766, "35": 0.12823, "40": 0.12721, "45": 0.12802, "50": 0.12653, "55": 0.13387, "60": 0.12431, "65": 0.12501, "70": 0.12535, "75": 0.12473, "80": 0.12431, "85": 0.12451, "90": 0.12436, "95": 0.12406, "100": 0.12423}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84449, + "13": 10.86322, + "14": 10.86353, + "15": 10.79981, + "16": 10.79262, + "17": 10.77477, + "18": 10.80157, + "19": 10.79148, + "20": 10.70508, + "21": 10.68176, + "22": 10.56548, + "23": 10.70147, + "24": 10.57889, + "25": 10.53597, + "26": 10.60742, + "27": 10.59423, + "28": 10.56119, + "29": 10.57569, + "30": 10.35474, + "31": 10.12616, + "32": 10.46566, + "33": 10.45233, + "34": 10.22493, + "35": 10.27091, + "36": 10.22168, + "37": 10.33936, + "38": 10.18641, + "39": 10.39431, + "40": 10.07792, + "41": 10.13872, + "42": 10.20182, + "43": 9.83818, + "44": 9.94274, + "45": 9.82303, + "46": 9.82185, + "47": 10.13443, + "48": 9.84098, + "49": 9.52095, + "50": 9.90109, + "51": 9.83457, + "52": 9.73232, + "53": 10.0488, + "54": 9.93895, + "55": 9.863, + "56": 9.613, + "57": 9.46966, + "58": 9.81135, + "59": 9.57107, + "60": 9.48155, + "61": 9.6788, + "62": 9.96581, + "63": 9.35273, + "64": 9.75648, + "65": 8.93771, + "66": 9.68153, + "67": 9.35671, + "68": 9.76807, + "69": 9.7739, + "70": 9.71016, + "71": 9.60009, + "72": 9.56793, + "73": 9.4774, + "74": 8.93177, + "75": 9.4072, + "76": 9.06849, + "77": 10.0464, + "78": 9.70988, + "79": 9.35733, + "80": 9.38975, + "81": 9.4662, + "82": 9.68058, + "83": 9.2914, + "84": 9.40191, + "85": 9.59735, + "86": 9.06209, + "87": 9.57922, + "88": 9.73259, + "89": 9.58836, + "90": 9.80354, + "91": 9.31991, + "92": 9.35011, + "93": 9.06382, + "94": 8.81909, + "95": 9.50568, + "96": 9.51071, + "97": 9.29241, + "98": 9.65578, + "99": 8.87401, + "100": 9.38833 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1966.0, + "12": 1767.0, + "13": 1876.0, + "14": 1881.0, + "15": 1928.0, + "16": 1875.0, + "17": 1790.0, + "18": 1736.0, + "19": 1804.0, + "20": 1690.0, + "21": 2008.0, + "22": 1765.0, + "23": 2073.0, + "24": 1618.0, + "25": 1720.0, + "26": 1807.0, + "27": 1861.0, + "28": 2026.0, + "29": 1982.0, + "30": 1981.0, + "31": 1688.0, + "32": 1913.0, + "33": 2123.0, + "34": 1893.0, + "35": 2007.0, + "36": 1987.0, + "37": 2334.0, + "38": 2223.0, + "39": 2417.0, + "40": 2370.0, + "41": 2352.0, + "42": 2269.0, + "43": 1967.0, + "44": 2183.0, + "45": 2150.0, + "46": 2350.0, + "47": 2555.0, + "48": 2463.0, + "49": 2326.0, + "50": 2270.0, + "51": 2508.0, + "52": 2495.0, + "53": 2856.0, + "54": 2692.0, + "55": 2482.0, + "56": 2614.0, + "57": 2283.0, + "58": 2894.0, + "59": 2659.0, + "60": 2561.0, + "61": 3006.0, + "62": 2671.0, + "63": 2488.0, + "64": 3092.0, + "65": 2622.0, + "66": 3108.0, + "67": 2741.0, + "68": 2942.0, + "69": 2983.0, + "70": 3347.0, + "71": 3034.0, + "72": 2438.0, + "73": 3075.0, + "74": 1931.0, + "75": 2722.0, + "76": 2960.0, + "77": 3387.0, + "78": 3268.0, + "79": 3079.0, + "80": 3404.0, + "81": 3674.0, + "82": 3192.0, + "83": 2791.0, + "84": 3224.0, + "85": 3237.0, + "86": 2646.0, + "87": 3840.0, + "88": 3114.0, + "89": 3410.0, + "90": 3184.0, + "91": 3073.0, + "92": 3396.0, + "93": 2711.0, + "94": 3530.0, + "95": 3387.0, + "96": 3530.0, + "97": 3277.0, + "98": 3775.0, + "99": 3421.0, + "100": 3350.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1234585088.0, + "2": 1234585088.0, + "3": 1234585088.0, + "4": 1234585088.0, + "5": 1234585088.0, + "6": 1234585088.0, + "7": 1234585088.0, + "8": 1234585088.0, + "9": 1234585088.0, + "10": 1234585088.0, + "11": 1234585088.0, + "12": 1234585088.0, + "13": 1234585088.0, + "14": 1234585088.0, + "15": 1234585088.0, + "16": 1234585088.0, + "17": 1234585088.0, + "18": 1234585088.0, + "19": 1234585088.0, + "20": 1234585088.0, + "21": 1234585088.0, + "22": 1234585088.0, + "23": 1234585088.0, + "24": 1234585088.0, + "25": 1234585088.0, + "26": 1234585088.0, + "27": 1234585088.0, + "28": 1234585088.0, + "29": 1234585088.0, + "30": 1234585088.0, + "31": 1234585088.0, + "32": 1234585088.0, + "33": 1234585088.0, + "34": 1234585088.0, + "35": 1234585088.0, + "36": 1234585088.0, + "37": 1234585088.0, + "38": 1234585088.0, + "39": 1234585088.0, + "40": 1234585088.0, + "41": 1234585088.0, + "42": 1234585088.0, + "43": 1234585088.0, + "44": 1234585088.0, + "45": 1234585088.0, + "46": 1234585088.0, + "47": 1234585088.0, + "48": 1234585088.0, + "49": 1234585088.0, + "50": 1234585088.0, + "51": 1234585088.0, + "52": 1234585088.0, + "53": 1234585088.0, + "54": 1234585088.0, + "55": 1234585088.0, + "56": 1234585088.0, + "57": 1234585088.0, + "58": 1234585088.0, + "59": 1234585088.0, + "60": 1234585088.0, + "61": 1234585088.0, + "62": 1234585088.0, + "63": 1234585088.0, + "64": 1234585088.0, + "65": 1234585088.0, + "66": 1234585088.0, + "67": 1234585088.0, + "68": 1234585088.0, + "69": 1234585088.0, + "70": 1234585088.0, + "71": 1234585088.0, + "72": 1234585088.0, + "73": 1234585088.0, + "74": 1234585088.0, + "75": 1234585088.0, + "76": 1234585088.0, + "77": 1234585088.0, + "78": 1234585088.0, + "79": 1234585088.0, + "80": 1234585088.0, + "81": 1234585088.0, + "82": 1234585088.0, + "83": 1234585088.0, + "84": 1234585088.0, + "85": 1234585088.0, + "86": 1234585088.0, + "87": 1234585088.0, + "88": 1234585088.0, + "89": 1234585088.0, + "90": 1234585088.0, + "91": 1234585088.0, + "92": 1234585088.0, + "93": 1234585088.0, + "94": 1234585088.0, + "95": 1234585088.0, + "96": 1234585088.0, + "97": 1234585088.0, + "98": 1234585088.0, + "99": 1234585088.0, + "100": 1234585088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1984492544.0, + "2": 2536167424.0, + "3": 2536167424.0, + "4": 2536167424.0, + "5": 2536167424.0, + "6": 2536167424.0, + "7": 2536167424.0, + "8": 2536167424.0, + "9": 2536167424.0, + "10": 2536167424.0, + "11": 2536167424.0, + "12": 2536167424.0, + "13": 2536167424.0, + "14": 2536167424.0, + "15": 2536167424.0, + "16": 2536167424.0, + "17": 2536167424.0, + "18": 2536167424.0, + "19": 2536167424.0, + "20": 2536167424.0, + "21": 2536167424.0, + "22": 2536167424.0, + "23": 2536167424.0, + "24": 2536167424.0, + "25": 2536167424.0, + "26": 2536167424.0, + "27": 2536167424.0, + "28": 2536167424.0, + "29": 2536167424.0, + "30": 2536167424.0, + "31": 2536167424.0, + "32": 2536167424.0, + "33": 2536167424.0, + "34": 2536167424.0, + "35": 2536167424.0, + "36": 2536167424.0, + "37": 2536167424.0, + "38": 2536167424.0, + "39": 2536167424.0, + "40": 2536167424.0, + "41": 2536167424.0, + "42": 2536167424.0, + "43": 2536167424.0, + "44": 2536167424.0, + "45": 2536167424.0, + "46": 2536167424.0, + "47": 2536167424.0, + "48": 2536167424.0, + "49": 2536167424.0, + "50": 2536167424.0, + "51": 2536167424.0, + "52": 2536167424.0, + "53": 2536167424.0, + "54": 2536167424.0, + "55": 2536167424.0, + "56": 2536167424.0, + "57": 2536167424.0, + "58": 2536167424.0, + "59": 2536167424.0, + "60": 2536167424.0, + "61": 2536167424.0, + "62": 2536167424.0, + "63": 2536167424.0, + "64": 2536167424.0, + "65": 2536167424.0, + "66": 2536167424.0, + "67": 2536167424.0, + "68": 2536167424.0, + "69": 2536167424.0, + "70": 2536167424.0, + "71": 2536167424.0, + "72": 2536167424.0, + "73": 2536167424.0, + "74": 2536167424.0, + "75": 2536167424.0, + "76": 2536167424.0, + "77": 2536167424.0, + "78": 2536167424.0, + "79": 2536167424.0, + "80": 2536167424.0, + "81": 2536167424.0, + "82": 2536167424.0, + "83": 2536167424.0, + "84": 2536167424.0, + "85": 2536167424.0, + "86": 2536167424.0, + "87": 2536167424.0, + "88": 2536167424.0, + "89": 2536167424.0, + "90": 2536167424.0, + "91": 2536167424.0, + "92": 2536167424.0, + "93": 2536167424.0, + "94": 2536167424.0, + "95": 2536167424.0, + "96": 2536167424.0, + "97": 2536167424.0, + "98": 2536167424.0, + "99": 2536167424.0, + "100": 2536167424.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.43734, + "2": 0.14648, + "3": 0.12542, + "4": 0.12603, + "5": 0.12388, + "6": 0.12524, + "7": 0.12279, + "8": 0.1239, + "9": 0.12244, + "10": 0.12336, + "11": 0.12345, + "12": 0.12322, + "13": 0.12318, + "14": 0.12381, + "15": 0.12343, + "16": 0.12319, + "17": 0.12276, + "18": 0.12324, + "19": 0.12355, + "20": 0.12315, + "21": 0.12294, + "22": 0.12326, + "23": 0.12303, + "24": 0.12294, + "25": 0.12286, + "26": 0.12388, + "27": 0.12341, + "28": 0.12301, + "29": 0.12267, + "30": 0.12315, + "31": 0.12371, + "32": 0.12359, + "33": 0.12298, + "34": 0.12283, + "35": 0.12266, + "36": 0.12356, + "37": 0.12377, + "38": 0.12388, + "39": 0.12525, + "40": 0.12501, + "41": 0.12357, + "42": 0.12376, + "43": 0.12304, + "44": 0.12342, + "45": 0.12284, + "46": 0.12332, + "47": 0.12324, + "48": 0.12279, + "49": 0.12276, + "50": 0.12391, + "51": 0.12862, + "52": 0.12214, + "53": 0.12006, + "54": 0.12101, + "55": 0.12062, + "56": 0.12088, + "57": 0.121, + "58": 0.12034, + "59": 0.12049, + "60": 0.12066, + "61": 0.11974, + "62": 0.11979, + "63": 0.12196, + "64": 0.12149, + "65": 0.12119, + "66": 0.12067, + "67": 0.12079, + "68": 0.12104, + "69": 0.12025, + "70": 0.12059, + "71": 0.12069, + "72": 0.12102, + "73": 0.12115, + "74": 0.1208, + "75": 0.12051, + "76": 0.12011, + "77": 0.11958, + "78": 0.12095, + "79": 0.11983, + "80": 0.12106, + "81": 0.1203, + "82": 0.12062, + "83": 0.12021, + "84": 0.12036, + "85": 0.12053, + "86": 0.12119, + "87": 0.12057, + "88": 0.12092, + "89": 0.12271, + "90": 0.12095, + "91": 0.1204, + "92": 0.12052, + "93": 0.12075, + "94": 0.12025, + "95": 0.12129, + "96": 0.12087, + "97": 0.12098, + "98": 0.12136, + "99": 0.12046, + "100": 0.12064 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json new file mode 100644 index 00000000000..02ac3f23ca4 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84865, + "2": 10.84835, + "3": 10.84054, + "4": 10.82381, + "5": 10.86283, + "6": 10.87554, + "7": 10.87089, + "8": 10.85554, + "9": 10.86811, + "10": 10.82287, + "11": 10.90334, + "12": 10.87985, + "13": 10.8831, + "14": 10.89653, + "15": 10.81406, + "16": 10.81973, + "17": 10.79918, + "18": 10.82428, + "19": 10.8195, + "20": 10.71958, + "21": 10.67864, + "22": 10.53625, + "23": 10.72125, + "24": 10.57435, + "25": 10.52997, + "26": 10.60096, + "27": 10.61502, + "28": 10.57231, + "29": 10.58614, + "30": 10.33749, + "31": 10.06516, + "32": 10.46436, + "33": 10.43612, + "34": 10.17278, + "35": 10.24173, + "36": 10.19042, + "37": 10.32282, + "38": 10.14881, + "39": 10.37709, + "40": 10.05124, + "41": 10.11355, + "42": 10.17253, + "43": 9.76298, + "44": 9.89293, + "45": 9.7664, + "46": 9.7601, + "47": 10.09424, + "48": 9.78753, + "49": 9.454, + "50": 9.8548, + "51": 9.79157, + "52": 9.68731, + "53": 10.02181, + "54": 9.90398, + "55": 9.82389, + "56": 9.57081, + "57": 9.40818, + "58": 9.77678, + "59": 9.52729, + "60": 9.44284, + "61": 9.64071, + "62": 9.94046, + "63": 9.31099, + "64": 9.72506, + "65": 8.8916, + "66": 9.6525, + "67": 9.31718, + "68": 9.73957, + "69": 9.74304, + "70": 9.67942, + "71": 9.56228, + "72": 9.53149, + "73": 9.44531, + "74": 8.88431, + "75": 9.3677, + "76": 9.02482, + "77": 10.01647, + "78": 9.6813, + "79": 9.32719, + "80": 9.3577, + "81": 9.43335, + "82": 9.64804, + "83": 9.25573, + "84": 9.36738, + "85": 9.56091, + "86": 9.03567, + "87": 9.54622, + "88": 9.70041, + "89": 9.54992, + "90": 9.77126, + "91": 9.28801, + "92": 9.31055, + "93": 9.03195, + "94": 8.78121, + "95": 9.48115, + "96": 9.4759, + "97": 9.2489, + "98": 9.61705, + "99": 8.8368, + "100": 9.35043 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1658.0, + "2": 1892.0, + "3": 1844.0, + "4": 1851.0, + "5": 1895.0, + "6": 2058.0, + "7": 1902.0, + "8": 1808.0, + "9": 1894.0, + "10": 1583.0, + "11": 2044.0, + "12": 1889.0, + "13": 2101.0, + "14": 2062.0, + "15": 1981.0, + "16": 1922.0, + "17": 1845.0, + "18": 1842.0, + "19": 1890.0, + "20": 1769.0, + "21": 2048.0, + "22": 1800.0, + "23": 2134.0, + "24": 1822.0, + "25": 1861.0, + "26": 1864.0, + "27": 1886.0, + "28": 2050.0, + "29": 2009.0, + "30": 2118.0, + "31": 1688.0, + "32": 2112.0, + "33": 2153.0, + "34": 1963.0, + "35": 2014.0, + "36": 2023.0, + "37": 2222.0, + "38": 2239.0, + "39": 2367.0, + "40": 2161.0, + "41": 2422.0, + "42": 2240.0, + "43": 2116.0, + "44": 2343.0, + "45": 2127.0, + "46": 2189.0, + "47": 2411.0, + "48": 2347.0, + "49": 2271.0, + "50": 2345.0, + "51": 2482.0, + "52": 2570.0, + "53": 2835.0, + "54": 2589.0, + "55": 2450.0, + "56": 2744.0, + "57": 2429.0, + "58": 2684.0, + "59": 2748.0, + "60": 2464.0, + "61": 2995.0, + "62": 2518.0, + "63": 2570.0, + "64": 2843.0, + "65": 2648.0, + "66": 2842.0, + "67": 2954.0, + "68": 2833.0, + "69": 3027.0, + "70": 2993.0, + "71": 3010.0, + "72": 2597.0, + "73": 3002.0, + "74": 2325.0, + "75": 2882.0, + "76": 3143.0, + "77": 3062.0, + "78": 3272.0, + "79": 3303.0, + "80": 3280.0, + "81": 3517.0, + "82": 3283.0, + "83": 2834.0, + "84": 3365.0, + "85": 3288.0, + "86": 2562.0, + "87": 3493.0, + "88": 3388.0, + "89": 3102.0, + "90": 3230.0, + "91": 3154.0, + "92": 3263.0, + "93": 2967.0, + "94": 3520.0, + "95": 3175.0, + "96": 3317.0, + "97": 2999.0, + "98": 3549.0, + "99": 3248.0, + "100": 3227.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 892293120.0, + "2": 892293120.0, + "3": 892293120.0, + "4": 892293120.0, + "5": 892293120.0, + "6": 892293120.0, + "7": 892293120.0, + "8": 892293120.0, + "9": 892293120.0, + "10": 892293120.0, + "11": 892293120.0, + "12": 892293120.0, + "13": 892293120.0, + "14": 892293120.0, + "15": 892293120.0, + "16": 892293120.0, + "17": 892293120.0, + "18": 892293120.0, + "19": 892293120.0, + "20": 892293120.0, + "21": 892293120.0, + "22": 892293120.0, + "23": 892293120.0, + "24": 892293120.0, + "25": 892293120.0, + "26": 892293120.0, + "27": 892293120.0, + "28": 892293120.0, + "29": 892293120.0, + "30": 892293120.0, + "31": 892293120.0, + "32": 892293120.0, + "33": 892293120.0, + "34": 892293120.0, + "35": 892293120.0, + "36": 892293120.0, + "37": 892293120.0, + "38": 892293120.0, + "39": 892293120.0, + "40": 892293120.0, + "41": 892293120.0, + "42": 892293120.0, + "43": 892293120.0, + "44": 892293120.0, + "45": 892293120.0, + "46": 892293120.0, + "47": 892293120.0, + "48": 892293120.0, + "49": 892293120.0, + "50": 892293120.0, + "51": 892293120.0, + "52": 892293120.0, + "53": 892293120.0, + "54": 892293120.0, + "55": 892293120.0, + "56": 892293120.0, + "57": 892293120.0, + "58": 892293120.0, + "59": 892293120.0, + "60": 892293120.0, + "61": 892293120.0, + "62": 892293120.0, + "63": 892293120.0, + "64": 892293120.0, + "65": 892293120.0, + "66": 892293120.0, + "67": 892293120.0, + "68": 892293120.0, + "69": 892293120.0, + "70": 892293120.0, + "71": 892293120.0, + "72": 892293120.0, + "73": 892293120.0, + "74": 892293120.0, + "75": 892293120.0, + "76": 892293120.0, + "77": 892293120.0, + "78": 892293120.0, + "79": 892293120.0, + "80": 892293120.0, + "81": 892293120.0, + "82": 892293120.0, + "83": 892293120.0, + "84": 892293120.0, + "85": 892293120.0, + "86": 892293120.0, + "87": 892293120.0, + "88": 892293120.0, + "89": 892293120.0, + "90": 892293120.0, + "91": 892293120.0, + "92": 892293120.0, + "93": 892293120.0, + "94": 892293120.0, + "95": 892293120.0, + "96": 892293120.0, + "97": 892293120.0, + "98": 892293120.0, + "99": 892293120.0, + "100": 892293120.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2236674048.0, + "2": 2599285760.0, + "3": 2599285760.0, + "4": 2599285760.0, + "5": 2599285760.0, + "6": 2599285760.0, + "7": 2599285760.0, + "8": 2599285760.0, + "9": 2599285760.0, + "10": 2599285760.0, + "11": 2599285760.0, + "12": 2599285760.0, + "13": 2599285760.0, + "14": 2599285760.0, + "15": 2599285760.0, + "16": 2599285760.0, + "17": 2599285760.0, + "18": 2599285760.0, + "19": 2599285760.0, + "20": 2599285760.0, + "21": 2599285760.0, + "22": 2599285760.0, + "23": 2599285760.0, + "24": 2599285760.0, + "25": 2599285760.0, + "26": 2599285760.0, + "27": 2599285760.0, + "28": 2599285760.0, + "29": 2599285760.0, + "30": 2599285760.0, + "31": 2599285760.0, + "32": 2599285760.0, + "33": 2599285760.0, + "34": 2599285760.0, + "35": 2599285760.0, + "36": 2599285760.0, + "37": 2599285760.0, + "38": 2599285760.0, + "39": 2599285760.0, + "40": 2599285760.0, + "41": 2599285760.0, + "42": 2599285760.0, + "43": 2599285760.0, + "44": 2599285760.0, + "45": 2599285760.0, + "46": 2599285760.0, + "47": 2599285760.0, + "48": 2599285760.0, + "49": 2599285760.0, + "50": 2599285760.0, + "51": 2599285760.0, + "52": 2599285760.0, + "53": 2599285760.0, + "54": 2599285760.0, + "55": 2599285760.0, + "56": 2599285760.0, + "57": 2599285760.0, + "58": 2599285760.0, + "59": 2599285760.0, + "60": 2599285760.0, + "61": 2599285760.0, + "62": 2599285760.0, + "63": 2599285760.0, + "64": 2599285760.0, + "65": 2599285760.0, + "66": 2599285760.0, + "67": 2599285760.0, + "68": 2599285760.0, + "69": 2599285760.0, + "70": 2599285760.0, + "71": 2599285760.0, + "72": 2599285760.0, + "73": 2599285760.0, + "74": 2599285760.0, + "75": 2599285760.0, + "76": 2599285760.0, + "77": 2599285760.0, + "78": 2599285760.0, + "79": 2599285760.0, + "80": 2599285760.0, + "81": 2599285760.0, + "82": 2599285760.0, + "83": 2599285760.0, + "84": 2599285760.0, + "85": 2599285760.0, + "86": 2599285760.0, + "87": 2599285760.0, + "88": 2599285760.0, + "89": 2599285760.0, + "90": 2599285760.0, + "91": 2599285760.0, + "92": 2599285760.0, + "93": 2599285760.0, + "94": 2599285760.0, + "95": 2599285760.0, + "96": 2599285760.0, + "97": 2599285760.0, + "98": 2599285760.0, + "99": 2599285760.0, + "100": 2599285760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.33769, + "2": 0.18532, + "3": 0.14816, + "4": 0.14635, + "5": 0.14643, + "6": 0.14673, + "7": 0.1465, + "8": 0.14731, + "9": 0.14559, + "10": 0.1459, + "11": 0.14535, + "12": 0.14613, + "13": 0.14618, + "14": 0.14642, + "15": 0.1462, + "16": 0.14548, + "17": 0.14502, + "18": 0.1449, + "19": 0.145, + "20": 0.14696, + "21": 0.14496, + "22": 0.14536, + "23": 0.14489, + "24": 0.14464, + "25": 0.14477, + "26": 0.14552, + "27": 0.14534, + "28": 0.14506, + "29": 0.14487, + "30": 0.14509, + "31": 0.14642, + "32": 0.14499, + "33": 0.14538, + "34": 0.14461, + "35": 0.14541, + "36": 0.14464, + "37": 0.14539, + "38": 0.14569, + "39": 0.14515, + "40": 0.14583, + "41": 0.14836, + "42": 0.14589, + "43": 0.14625, + "44": 0.14559, + "45": 0.14588, + "46": 0.14644, + "47": 0.146, + "48": 0.1459, + "49": 0.14609, + "50": 0.14597, + "51": 0.15206, + "52": 0.1459, + "53": 0.1452, + "54": 0.14745, + "55": 0.14578, + "56": 0.14459, + "57": 0.14524, + "58": 0.14545, + "59": 0.14527, + "60": 0.14448, + "61": 0.14539, + "62": 0.14462, + "63": 0.14474, + "64": 0.1447, + "65": 0.14564, + "66": 0.14463, + "67": 0.14466, + "68": 0.14483, + "69": 0.14562, + "70": 0.1456, + "71": 0.14516, + "72": 0.14481, + "73": 0.14539, + "74": 0.14568, + "75": 0.14464, + "76": 0.14465, + "77": 0.14427, + "78": 0.14541, + "79": 0.1445, + "80": 0.14535, + "81": 0.14526, + "82": 0.14617, + "83": 0.14445, + "84": 0.14483, + "85": 0.14457, + "86": 0.14459, + "87": 0.14462, + "88": 0.14433, + "89": 0.14514, + "90": 0.14416, + "91": 0.14667, + "92": 0.14432, + "93": 0.14551, + "94": 0.14453, + "95": 0.14488, + "96": 0.14441, + "97": 0.14545, + "98": 0.14459, + "99": 0.14481, + "100": 0.14918 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json index d30c8d5b5e6..48aee8d379f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.84474, - "5": 10.86418, - "10": 10.82155, - "15": 10.81195, - "20": 10.71872, - "25": 10.53036, - "30": 10.3358, - "35": 10.24081, - "40": 10.05008, - "45": 9.76762, - "50": 9.85505, - "55": 9.82465, - "60": 9.44306, - "65": 8.89104, - "70": 9.67902, - "75": 9.36836, - "80": 9.35799, - "85": 9.56032, - "90": 9.77055, - "95": 9.48101, - "100": 9.34997 + "1": 10.84865, + "2": 10.84835, + "3": 10.84054, + "4": 10.82381, + "5": 10.86283, + "6": 10.87554, + "7": 10.87089, + "8": 10.85554, + "9": 10.86811, + "10": 10.82287, + "11": 10.90334, + "12": 10.87985, + "13": 10.8831, + "14": 10.89653, + "15": 10.81406, + "16": 10.81973, + "17": 10.79918, + "18": 10.82428, + "19": 10.8195, + "20": 10.71958, + "21": 10.67864, + "22": 10.53625, + "23": 10.72125, + "24": 10.57435, + "25": 10.52997, + "26": 10.60096, + "27": 10.61502, + "28": 10.57231, + "29": 10.58614, + "30": 10.33749, + "31": 10.06516, + "32": 10.46436, + "33": 10.43612, + "34": 10.17278, + "35": 10.24173, + "36": 10.19042, + "37": 10.32282, + "38": 10.14881, + "39": 10.37709, + "40": 10.05124, + "41": 10.11355, + "42": 10.17253, + "43": 9.76298, + "44": 9.89293, + "45": 9.7664, + "46": 9.7601, + "47": 10.09424, + "48": 9.78753, + "49": 9.454, + "50": 9.8548, + "51": 9.79157, + "52": 9.68731, + "53": 10.02181, + "54": 9.90398, + "55": 9.82389, + "56": 9.57081, + "57": 9.40818, + "58": 9.77678, + "59": 9.52729, + "60": 9.44284, + "61": 9.64071, + "62": 9.94046, + "63": 9.31099, + "64": 9.72506, + "65": 8.8916, + "66": 9.6525, + "67": 9.31718, + "68": 9.73957, + "69": 9.74304, + "70": 9.67942, + "71": 9.56228, + "72": 9.53149, + "73": 9.44531, + "74": 8.88431, + "75": 9.3677, + "76": 9.02482, + "77": 10.01647, + "78": 9.6813, + "79": 9.32719, + "80": 9.3577, + "81": 9.43335, + "82": 9.64804, + "83": 9.25573, + "84": 9.36738, + "85": 9.56091, + "86": 9.03567, + "87": 9.54622, + "88": 9.70041, + "89": 9.54992, + "90": 9.77126, + "91": 9.28801, + "92": 9.31055, + "93": 9.03195, + "94": 8.78121, + "95": 9.48115, + "96": 9.4759, + "97": 9.2489, + "98": 9.61705, + "99": 8.8368, + "100": 9.35043 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1776.0, - "5": 2128.0, - "10": 1615.0, - "15": 2021.0, - "20": 1775.0, - "25": 1916.0, - "30": 2029.0, - "35": 2107.0, - "40": 2174.0, - "45": 2110.0, - "50": 2363.0, - "55": 2460.0, - "60": 2462.0, - "65": 2724.0, - "70": 2952.0, - "75": 2823.0, - "80": 3222.0, - "85": 3314.0, - "90": 3087.0, - "95": 3146.0, - "100": 3331.0 + "1": 1658.0, + "2": 1892.0, + "3": 1844.0, + "4": 1851.0, + "5": 1895.0, + "6": 2058.0, + "7": 1902.0, + "8": 1808.0, + "9": 1894.0, + "10": 1583.0, + "11": 2044.0, + "12": 1889.0, + "13": 2101.0, + "14": 2062.0, + "15": 1981.0, + "16": 1922.0, + "17": 1845.0, + "18": 1842.0, + "19": 1890.0, + "20": 1769.0, + "21": 2048.0, + "22": 1800.0, + "23": 2134.0, + "24": 1822.0, + "25": 1861.0, + "26": 1864.0, + "27": 1886.0, + "28": 2050.0, + "29": 2009.0, + "30": 2118.0, + "31": 1688.0, + "32": 2112.0, + "33": 2153.0, + "34": 1963.0, + "35": 2014.0, + "36": 2023.0, + "37": 2222.0, + "38": 2239.0, + "39": 2367.0, + "40": 2161.0, + "41": 2422.0, + "42": 2240.0, + "43": 2116.0, + "44": 2343.0, + "45": 2127.0, + "46": 2189.0, + "47": 2411.0, + "48": 2347.0, + "49": 2271.0, + "50": 2345.0, + "51": 2482.0, + "52": 2570.0, + "53": 2835.0, + "54": 2589.0, + "55": 2450.0, + "56": 2744.0, + "57": 2429.0, + "58": 2684.0, + "59": 2748.0, + "60": 2464.0, + "61": 2995.0, + "62": 2518.0, + "63": 2570.0, + "64": 2843.0, + "65": 2648.0, + "66": 2842.0, + "67": 2954.0, + "68": 2833.0, + "69": 3027.0, + "70": 2993.0, + "71": 3010.0, + "72": 2597.0, + "73": 3002.0, + "74": 2325.0, + "75": 2882.0, + "76": 3143.0, + "77": 3062.0, + "78": 3272.0, + "79": 3303.0, + "80": 3280.0, + "81": 3517.0, + "82": 3283.0, + "83": 2834.0, + "84": 3365.0, + "85": 3288.0, + "86": 2562.0, + "87": 3493.0, + "88": 3388.0, + "89": 3102.0, + "90": 3230.0, + "91": 3154.0, + "92": 3263.0, + "93": 2967.0, + "94": 3520.0, + "95": 3175.0, + "96": 3317.0, + "97": 2999.0, + "98": 3549.0, + "99": 3248.0, + "100": 3227.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 888098304.0, - "5": 888098304.0, - "10": 888098304.0, - "15": 888098304.0, - "20": 888098304.0, - "25": 888098304.0, - "30": 888098304.0, - "35": 888098304.0, - "40": 888098304.0, - "45": 888098304.0, - "50": 888098304.0, - "55": 888098304.0, - "60": 888098304.0, - "65": 888098304.0, - "70": 888098304.0, - "75": 888098304.0, - "80": 888098304.0, - "85": 888098304.0, - "90": 888098304.0, - "95": 888098304.0, - "100": 888098304.0 + "1": 892293120.0, + "2": 892293120.0, + "3": 892293120.0, + "4": 892293120.0, + "5": 892293120.0, + "6": 892293120.0, + "7": 892293120.0, + "8": 892293120.0, + "9": 892293120.0, + "10": 892293120.0, + "11": 892293120.0, + "12": 892293120.0, + "13": 892293120.0, + "14": 892293120.0, + "15": 892293120.0, + "16": 892293120.0, + "17": 892293120.0, + "18": 892293120.0, + "19": 892293120.0, + "20": 892293120.0, + "21": 892293120.0, + "22": 892293120.0, + "23": 892293120.0, + "24": 892293120.0, + "25": 892293120.0, + "26": 892293120.0, + "27": 892293120.0, + "28": 892293120.0, + "29": 892293120.0, + "30": 892293120.0, + "31": 892293120.0, + "32": 892293120.0, + "33": 892293120.0, + "34": 892293120.0, + "35": 892293120.0, + "36": 892293120.0, + "37": 892293120.0, + "38": 892293120.0, + "39": 892293120.0, + "40": 892293120.0, + "41": 892293120.0, + "42": 892293120.0, + "43": 892293120.0, + "44": 892293120.0, + "45": 892293120.0, + "46": 892293120.0, + "47": 892293120.0, + "48": 892293120.0, + "49": 892293120.0, + "50": 892293120.0, + "51": 892293120.0, + "52": 892293120.0, + "53": 892293120.0, + "54": 892293120.0, + "55": 892293120.0, + "56": 892293120.0, + "57": 892293120.0, + "58": 892293120.0, + "59": 892293120.0, + "60": 892293120.0, + "61": 892293120.0, + "62": 892293120.0, + "63": 892293120.0, + "64": 892293120.0, + "65": 892293120.0, + "66": 892293120.0, + "67": 892293120.0, + "68": 892293120.0, + "69": 892293120.0, + "70": 892293120.0, + "71": 892293120.0, + "72": 892293120.0, + "73": 892293120.0, + "74": 892293120.0, + "75": 892293120.0, + "76": 892293120.0, + "77": 892293120.0, + "78": 892293120.0, + "79": 892293120.0, + "80": 892293120.0, + "81": 892293120.0, + "82": 892293120.0, + "83": 892293120.0, + "84": 892293120.0, + "85": 892293120.0, + "86": 892293120.0, + "87": 892293120.0, + "88": 892293120.0, + "89": 892293120.0, + "90": 892293120.0, + "91": 892293120.0, + "92": 892293120.0, + "93": 892293120.0, + "94": 892293120.0, + "95": 892293120.0, + "96": 892293120.0, + "97": 892293120.0, + "98": 892293120.0, + "99": 892293120.0, + "100": 892293120.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 3216302592.0, - "5": 3575768576.0, - "10": 3575768576.0, - "15": 3575768576.0, - "20": 3575768576.0, - "25": 3575768576.0, - "30": 3575768576.0, - "35": 3575768576.0, - "40": 3575768576.0, - "45": 3575768576.0, - "50": 3575768576.0, - "55": 3575768576.0, - "60": 3575768576.0, - "65": 3575768576.0, - "70": 3575768576.0, - "75": 3575768576.0, - "80": 3575768576.0, - "85": 3575768576.0, - "90": 3575768576.0, - "95": 3575768576.0, - "100": 3575768576.0 + "1": 2236674048.0, + "2": 2599285760.0, + "3": 2599285760.0, + "4": 2599285760.0, + "5": 2599285760.0, + "6": 2599285760.0, + "7": 2599285760.0, + "8": 2599285760.0, + "9": 2599285760.0, + "10": 2599285760.0, + "11": 2599285760.0, + "12": 2599285760.0, + "13": 2599285760.0, + "14": 2599285760.0, + "15": 2599285760.0, + "16": 2599285760.0, + "17": 2599285760.0, + "18": 2599285760.0, + "19": 2599285760.0, + "20": 2599285760.0, + "21": 2599285760.0, + "22": 2599285760.0, + "23": 2599285760.0, + "24": 2599285760.0, + "25": 2599285760.0, + "26": 2599285760.0, + "27": 2599285760.0, + "28": 2599285760.0, + "29": 2599285760.0, + "30": 2599285760.0, + "31": 2599285760.0, + "32": 2599285760.0, + "33": 2599285760.0, + "34": 2599285760.0, + "35": 2599285760.0, + "36": 2599285760.0, + "37": 2599285760.0, + "38": 2599285760.0, + "39": 2599285760.0, + "40": 2599285760.0, + "41": 2599285760.0, + "42": 2599285760.0, + "43": 2599285760.0, + "44": 2599285760.0, + "45": 2599285760.0, + "46": 2599285760.0, + "47": 2599285760.0, + "48": 2599285760.0, + "49": 2599285760.0, + "50": 2599285760.0, + "51": 2599285760.0, + "52": 2599285760.0, + "53": 2599285760.0, + "54": 2599285760.0, + "55": 2599285760.0, + "56": 2599285760.0, + "57": 2599285760.0, + "58": 2599285760.0, + "59": 2599285760.0, + "60": 2599285760.0, + "61": 2599285760.0, + "62": 2599285760.0, + "63": 2599285760.0, + "64": 2599285760.0, + "65": 2599285760.0, + "66": 2599285760.0, + "67": 2599285760.0, + "68": 2599285760.0, + "69": 2599285760.0, + "70": 2599285760.0, + "71": 2599285760.0, + "72": 2599285760.0, + "73": 2599285760.0, + "74": 2599285760.0, + "75": 2599285760.0, + "76": 2599285760.0, + "77": 2599285760.0, + "78": 2599285760.0, + "79": 2599285760.0, + "80": 2599285760.0, + "81": 2599285760.0, + "82": 2599285760.0, + "83": 2599285760.0, + "84": 2599285760.0, + "85": 2599285760.0, + "86": 2599285760.0, + "87": 2599285760.0, + "88": 2599285760.0, + "89": 2599285760.0, + "90": 2599285760.0, + "91": 2599285760.0, + "92": 2599285760.0, + "93": 2599285760.0, + "94": 2599285760.0, + "95": 2599285760.0, + "96": 2599285760.0, + "97": 2599285760.0, + "98": 2599285760.0, + "99": 2599285760.0, + "100": 2599285760.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.26216, - "5": 0.17522, - "10": 0.17308, - "15": 0.17129, - "20": 0.17153, - "25": 0.17072, - "30": 0.17154, - "35": 0.17096, - "40": 0.16956, - "45": 0.17006, - "50": 0.16739, - "55": 0.17674, - "60": 0.17809, - "65": 0.17803, - "70": 0.17656, - "75": 0.17665, - "80": 0.17686, - "85": 0.17744, - "90": 0.17611, - "95": 0.17687, - "100": 0.1753 + "1": 9.51973, + "2": 0.20593, + "3": 0.14945, + "4": 0.14775, + "5": 0.14785, + "6": 0.14767, + "7": 0.14754, + "8": 0.14649, + "9": 0.14636, + "10": 0.14713, + "11": 0.14628, + "12": 0.14658, + "13": 0.14581, + "14": 0.14652, + "15": 0.14657, + "16": 0.14585, + "17": 0.14783, + "18": 0.1469, + "19": 0.14603, + "20": 0.14662, + "21": 0.14635, + "22": 0.1461, + "23": 0.14688, + "24": 0.14579, + "25": 0.14587, + "26": 0.14836, + "27": 0.14598, + "28": 0.1458, + "29": 0.14604, + "30": 0.14624, + "31": 0.14719, + "32": 0.14625, + "33": 0.14582, + "34": 0.14603, + "35": 0.14619, + "36": 0.14587, + "37": 0.14585, + "38": 0.14625, + "39": 0.14572, + "40": 0.14629, + "41": 0.14561, + "42": 0.14587, + "43": 0.14672, + "44": 0.14572, + "45": 0.14618, + "46": 0.14622, + "47": 0.14572, + "48": 0.14538, + "49": 0.14571, + "50": 0.1457, + "51": 0.1553, + "52": 0.14793, + "53": 0.14797, + "54": 0.14774, + "55": 0.14702, + "56": 0.15765, + "57": 0.1544, + "58": 0.15368, + "59": 0.15399, + "60": 0.15366, + "61": 0.15362, + "62": 0.15351, + "63": 0.15339, + "64": 0.15353, + "65": 0.15154, + "66": 0.14531, + "67": 0.14661, + "68": 0.14599, + "69": 0.14546, + "70": 0.14633, + "71": 0.14568, + "72": 0.1461, + "73": 0.14601, + "74": 0.1469, + "75": 0.14561, + "76": 0.14575, + "77": 0.14581, + "78": 0.14634, + "79": 0.14619, + "80": 0.14627, + "81": 0.146, + "82": 0.14559, + "83": 0.14618, + "84": 0.14683, + "85": 0.14582, + "86": 0.1462, + "87": 0.14574, + "88": 0.14574, + "89": 0.14516, + "90": 0.14556, + "91": 0.146, + "92": 0.14702, + "93": 0.14541, + "94": 0.14625, + "95": 0.14586, + "96": 0.1455, + "97": 0.14559, + "98": 0.14614, + "99": 0.15005, + "100": 0.14598 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json new file mode 100644 index 00000000000..92fea79bc3d --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79332, + "2": 10.80674, + "3": 10.8069, + "4": 10.77377, + "5": 10.84763, + "6": 10.86617, + "7": 10.8277, + "8": 10.81226, + "9": 10.83507, + "10": 10.77007, + "11": 10.89243, + "12": 10.84575, + "13": 10.85895, + "14": 10.883, + "15": 10.79054, + "16": 10.78048, + "17": 10.75666, + "18": 10.79491, + "19": 10.79656, + "20": 10.68058, + "21": 10.66001, + "22": 10.50257, + "23": 10.7118, + "24": 10.55327, + "25": 10.50697, + "26": 10.58286, + "27": 10.58643, + "28": 10.55758, + "29": 10.56227, + "30": 10.33257, + "31": 10.08654, + "32": 10.44724, + "33": 10.44499, + "34": 10.19999, + "35": 10.25465, + "36": 10.19443, + "37": 10.32044, + "38": 10.16641, + "39": 10.3774, + "40": 10.05603, + "41": 10.13739, + "42": 10.19161, + "43": 9.80954, + "44": 9.93054, + "45": 9.80619, + "46": 9.81395, + "47": 10.12881, + "48": 9.82729, + "49": 9.51291, + "50": 9.89126, + "51": 9.84055, + "52": 9.73438, + "53": 10.05482, + "54": 9.94058, + "55": 9.87124, + "56": 9.61045, + "57": 9.46116, + "58": 9.81654, + "59": 9.57887, + "60": 9.48507, + "61": 9.68515, + "62": 9.97438, + "63": 9.36298, + "64": 9.76793, + "65": 8.93913, + "66": 9.68918, + "67": 9.36638, + "68": 9.77507, + "69": 9.78344, + "70": 9.72196, + "71": 9.60806, + "72": 9.57714, + "73": 9.48934, + "74": 8.94008, + "75": 9.40867, + "76": 9.08075, + "77": 10.05717, + "78": 9.72281, + "79": 9.36465, + "80": 9.39746, + "81": 9.47553, + "82": 9.6886, + "83": 9.30263, + "84": 9.41008, + "85": 9.60793, + "86": 9.07115, + "87": 9.58676, + "88": 9.74129, + "89": 9.5986, + "90": 9.81041, + "91": 9.33113, + "92": 9.35502, + "93": 9.07481, + "94": 8.82745, + "95": 9.51149, + "96": 9.51876, + "97": 9.30173, + "98": 9.66726, + "99": 8.88087, + "100": 9.39727 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1624.0, + "2": 1758.0, + "3": 1678.0, + "4": 1680.0, + "5": 1824.0, + "6": 1899.0, + "7": 1871.0, + "8": 1677.0, + "9": 1802.0, + "10": 1414.0, + "11": 1904.0, + "12": 1630.0, + "13": 1956.0, + "14": 1777.0, + "15": 1902.0, + "16": 1801.0, + "17": 1830.0, + "18": 1630.0, + "19": 1835.0, + "20": 1642.0, + "21": 1752.0, + "22": 1696.0, + "23": 2085.0, + "24": 1618.0, + "25": 1593.0, + "26": 1722.0, + "27": 1778.0, + "28": 2042.0, + "29": 1900.0, + "30": 2001.0, + "31": 1592.0, + "32": 1757.0, + "33": 2116.0, + "34": 1924.0, + "35": 1904.0, + "36": 1852.0, + "37": 2382.0, + "38": 2195.0, + "39": 2267.0, + "40": 2335.0, + "41": 2223.0, + "42": 2317.0, + "43": 2069.0, + "44": 2060.0, + "45": 2140.0, + "46": 2397.0, + "47": 2464.0, + "48": 2455.0, + "49": 2276.0, + "50": 2374.0, + "51": 2574.0, + "52": 2457.0, + "53": 2905.0, + "54": 2609.0, + "55": 2220.0, + "56": 2663.0, + "57": 2258.0, + "58": 2898.0, + "59": 2676.0, + "60": 2397.0, + "61": 3048.0, + "62": 2533.0, + "63": 2370.0, + "64": 2975.0, + "65": 2591.0, + "66": 3065.0, + "67": 2732.0, + "68": 2870.0, + "69": 2955.0, + "70": 3112.0, + "71": 2989.0, + "72": 2451.0, + "73": 2881.0, + "74": 1859.0, + "75": 2649.0, + "76": 3026.0, + "77": 3316.0, + "78": 3212.0, + "79": 3183.0, + "80": 3262.0, + "81": 3669.0, + "82": 3187.0, + "83": 2798.0, + "84": 3209.0, + "85": 3309.0, + "86": 2738.0, + "87": 3804.0, + "88": 2989.0, + "89": 3327.0, + "90": 3031.0, + "91": 2720.0, + "92": 2972.0, + "93": 2719.0, + "94": 3387.0, + "95": 3321.0, + "96": 3342.0, + "97": 3191.0, + "98": 3533.0, + "99": 3214.0, + "100": 3318.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 716834304.0, + "2": 716834304.0, + "3": 716834304.0, + "4": 716834304.0, + "5": 716834304.0, + "6": 716834304.0, + "7": 716834304.0, + "8": 716834304.0, + "9": 716834304.0, + "10": 716834304.0, + "11": 716834304.0, + "12": 716834304.0, + "13": 716834304.0, + "14": 716834304.0, + "15": 716834304.0, + "16": 716834304.0, + "17": 716834304.0, + "18": 716834304.0, + "19": 716834304.0, + "20": 716834304.0, + "21": 716834304.0, + "22": 716834304.0, + "23": 716834304.0, + "24": 716834304.0, + "25": 716834304.0, + "26": 716834304.0, + "27": 716834304.0, + "28": 716834304.0, + "29": 716834304.0, + "30": 716834304.0, + "31": 716834304.0, + "32": 716834304.0, + "33": 716834304.0, + "34": 716834304.0, + "35": 716834304.0, + "36": 716834304.0, + "37": 716834304.0, + "38": 716834304.0, + "39": 716834304.0, + "40": 716834304.0, + "41": 716834304.0, + "42": 716834304.0, + "43": 716834304.0, + "44": 716834304.0, + "45": 716834304.0, + "46": 716834304.0, + "47": 716834304.0, + "48": 716834304.0, + "49": 716834304.0, + "50": 716834304.0, + "51": 716834304.0, + "52": 716834304.0, + "53": 716834304.0, + "54": 716834304.0, + "55": 716834304.0, + "56": 716834304.0, + "57": 716834304.0, + "58": 716834304.0, + "59": 716834304.0, + "60": 716834304.0, + "61": 716834304.0, + "62": 716834304.0, + "63": 716834304.0, + "64": 716834304.0, + "65": 716834304.0, + "66": 716834304.0, + "67": 716834304.0, + "68": 716834304.0, + "69": 716834304.0, + "70": 716834304.0, + "71": 716834304.0, + "72": 716834304.0, + "73": 716834304.0, + "74": 716834304.0, + "75": 716834304.0, + "76": 716834304.0, + "77": 716834304.0, + "78": 716834304.0, + "79": 716834304.0, + "80": 716834304.0, + "81": 716834304.0, + "82": 716834304.0, + "83": 716834304.0, + "84": 716834304.0, + "85": 716834304.0, + "86": 716834304.0, + "87": 716834304.0, + "88": 716834304.0, + "89": 716834304.0, + "90": 716834304.0, + "91": 716834304.0, + "92": 716834304.0, + "93": 716834304.0, + "94": 716834304.0, + "95": 716834304.0, + "96": 716834304.0, + "97": 716834304.0, + "98": 716834304.0, + "99": 716834304.0, + "100": 716834304.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910424576.0, + "2": 2193074176.0, + "3": 2193074176.0, + "4": 2193074176.0, + "5": 2193074176.0, + "6": 2193074176.0, + "7": 2193074176.0, + "8": 2193074176.0, + "9": 2193074176.0, + "10": 2193074176.0, + "11": 2193074176.0, + "12": 2193074176.0, + "13": 2193074176.0, + "14": 2193074176.0, + "15": 2193074176.0, + "16": 2193074176.0, + "17": 2193074176.0, + "18": 2193074176.0, + "19": 2193074176.0, + "20": 2193074176.0, + "21": 2193074176.0, + "22": 2193074176.0, + "23": 2193074176.0, + "24": 2193074176.0, + "25": 2193074176.0, + "26": 2193074176.0, + "27": 2193074176.0, + "28": 2193074176.0, + "29": 2193074176.0, + "30": 2193074176.0, + "31": 2193074176.0, + "32": 2193074176.0, + "33": 2193074176.0, + "34": 2193074176.0, + "35": 2193074176.0, + "36": 2193074176.0, + "37": 2193074176.0, + "38": 2193074176.0, + "39": 2193074176.0, + "40": 2193074176.0, + "41": 2193074176.0, + "42": 2193074176.0, + "43": 2193074176.0, + "44": 2193074176.0, + "45": 2193074176.0, + "46": 2193074176.0, + "47": 2193074176.0, + "48": 2193074176.0, + "49": 2193074176.0, + "50": 2193074176.0, + "51": 2193074176.0, + "52": 2193074176.0, + "53": 2193074176.0, + "54": 2193074176.0, + "55": 2193074176.0, + "56": 2193074176.0, + "57": 2193074176.0, + "58": 2193074176.0, + "59": 2193074176.0, + "60": 2193074176.0, + "61": 2193074176.0, + "62": 2193074176.0, + "63": 2193074176.0, + "64": 2193074176.0, + "65": 2193074176.0, + "66": 2193074176.0, + "67": 2193074176.0, + "68": 2193074176.0, + "69": 2193074176.0, + "70": 2193074176.0, + "71": 2193074176.0, + "72": 2193074176.0, + "73": 2193074176.0, + "74": 2193074176.0, + "75": 2193074176.0, + "76": 2193074176.0, + "77": 2193074176.0, + "78": 2193074176.0, + "79": 2193074176.0, + "80": 2193074176.0, + "81": 2193074176.0, + "82": 2193074176.0, + "83": 2193074176.0, + "84": 2193074176.0, + "85": 2193074176.0, + "86": 2193074176.0, + "87": 2193074176.0, + "88": 2193074176.0, + "89": 2193074176.0, + "90": 2193074176.0, + "91": 2193074176.0, + "92": 2193074176.0, + "93": 2193074176.0, + "94": 2193074176.0, + "95": 2193074176.0, + "96": 2193074176.0, + "97": 2193074176.0, + "98": 2193074176.0, + "99": 2193074176.0, + "100": 2193074176.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.17541, + "2": 0.17628, + "3": 0.14823, + "4": 0.14828, + "5": 0.14489, + "6": 0.14681, + "7": 0.14589, + "8": 0.14567, + "9": 0.14899, + "10": 0.14748, + "11": 0.1469, + "12": 0.14571, + "13": 0.14519, + "14": 0.14594, + "15": 0.14553, + "16": 0.1461, + "17": 0.14672, + "18": 0.14616, + "19": 0.1455, + "20": 0.14588, + "21": 0.14801, + "22": 0.14714, + "23": 0.14721, + "24": 0.14624, + "25": 0.1462, + "26": 0.14548, + "27": 0.14684, + "28": 0.14765, + "29": 0.14671, + "30": 0.14515, + "31": 0.14617, + "32": 0.14666, + "33": 0.14596, + "34": 0.14868, + "35": 0.14573, + "36": 0.14694, + "37": 0.14585, + "38": 0.14605, + "39": 0.14599, + "40": 0.14558, + "41": 0.14673, + "42": 0.14745, + "43": 0.1456, + "44": 0.14744, + "45": 0.14524, + "46": 0.14572, + "47": 0.14533, + "48": 0.14632, + "49": 0.14734, + "50": 0.1453, + "51": 0.16371, + "52": 0.14839, + "53": 0.17852, + "54": 0.14579, + "55": 0.14651, + "56": 0.14872, + "57": 0.14723, + "58": 0.14775, + "59": 0.14896, + "60": 0.14649, + "61": 0.14672, + "62": 0.14696, + "63": 0.14572, + "64": 0.14639, + "65": 0.14739, + "66": 0.14722, + "67": 0.14732, + "68": 0.14566, + "69": 0.14664, + "70": 0.14693, + "71": 0.14641, + "72": 0.14742, + "73": 0.14691, + "74": 0.1482, + "75": 0.15006, + "76": 0.146, + "77": 0.14585, + "78": 0.14677, + "79": 0.14716, + "80": 0.14605, + "81": 0.14678, + "82": 0.14648, + "83": 0.14624, + "84": 0.14639, + "85": 0.14622, + "86": 0.14829, + "87": 0.14591, + "88": 0.14541, + "89": 0.14865, + "90": 0.14587, + "91": 0.14618, + "92": 0.14625, + "93": 0.14624, + "94": 0.14583, + "95": 0.14675, + "96": 0.14876, + "97": 0.14645, + "98": 0.14588, + "99": 0.14617, + "100": 0.14618 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json index 4d218d51297..1246b8727ef 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.79205, "5": 10.84695, "10": 10.77106, "15": 10.79093, "20": 10.68042, "25": 10.50715, "30": 10.33325, "35": 10.25545, "40": 10.05544, "45": 9.80575, "50": 9.89082, "55": 9.87063, "60": 9.48478, "65": 8.94022, "70": 9.72243, "75": 9.40907, "80": 9.3976, "85": 9.60746, "90": 9.81041, "95": 9.5116, "100": 9.39722}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1580.0, "5": 1901.0, "10": 1346.0, "15": 1926.0, "20": 1643.0, "25": 1683.0, "30": 1867.0, "35": 2020.0, "40": 2252.0, "45": 2243.0, "50": 2459.0, "55": 2291.0, "60": 2404.0, "65": 2474.0, "70": 3102.0, "75": 2603.0, "80": 3420.0, "85": 3388.0, "90": 2904.0, "95": 3333.0, "100": 3347.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 714736640.0, "5": 714736640.0, "10": 714736640.0, "15": 714736640.0, "20": 714736640.0, "25": 714736640.0, "30": 714736640.0, "35": 714736640.0, "40": 714736640.0, "45": 714736640.0, "50": 714736640.0, "55": 714736640.0, "60": 714736640.0, "65": 714736640.0, "70": 714736640.0, "75": 714736640.0, "80": 714736640.0, "85": 714736640.0, "90": 714736640.0, "95": 714736640.0, "100": 714736640.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2399714304.0, "5": 2681315328.0, "10": 2681315328.0, "15": 2681315328.0, "20": 2681315328.0, "25": 2681315328.0, "30": 2681315328.0, "35": 2681315328.0, "40": 2681315328.0, "45": 2681315328.0, "50": 2681315328.0, "55": 2681315328.0, "60": 2681315328.0, "65": 2681315328.0, "70": 2681315328.0, "75": 2681315328.0, "80": 2681315328.0, "85": 2681315328.0, "90": 2681315328.0, "95": 2681315328.0, "100": 2681315328.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 9.04394, "5": 0.16449, "10": 0.16258, "15": 0.16166, "20": 0.18479, "25": 0.17915, "30": 0.18008, "35": 0.16013, "40": 0.15723, "45": 0.15769, "50": 0.15699, "55": 0.15776, "60": 0.15787, "65": 0.15913, "70": 0.15699, "75": 0.15748, "80": 0.15897, "85": 0.15762, "90": 0.15743, "95": 0.15733, "100": 0.15822}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79332, + "2": 10.80674, + "3": 10.8069, + "4": 10.77377, + "5": 10.84763, + "6": 10.86617, + "7": 10.8277, + "8": 10.81226, + "9": 10.83507, + "10": 10.77007, + "11": 10.89243, + "12": 10.84575, + "13": 10.85895, + "14": 10.883, + "15": 10.79054, + "16": 10.78048, + "17": 10.75666, + "18": 10.79491, + "19": 10.79656, + "20": 10.68058, + "21": 10.66001, + "22": 10.50257, + "23": 10.7118, + "24": 10.55327, + "25": 10.50697, + "26": 10.58286, + "27": 10.58643, + "28": 10.55758, + "29": 10.56227, + "30": 10.33257, + "31": 10.08654, + "32": 10.44724, + "33": 10.44499, + "34": 10.19999, + "35": 10.25465, + "36": 10.19443, + "37": 10.32044, + "38": 10.16641, + "39": 10.3774, + "40": 10.05603, + "41": 10.13739, + "42": 10.19161, + "43": 9.80954, + "44": 9.93054, + "45": 9.80619, + "46": 9.81395, + "47": 10.12881, + "48": 9.82729, + "49": 9.51291, + "50": 9.89126, + "51": 9.84055, + "52": 9.73438, + "53": 10.05482, + "54": 9.94058, + "55": 9.87124, + "56": 9.61045, + "57": 9.46116, + "58": 9.81654, + "59": 9.57887, + "60": 9.48507, + "61": 9.68515, + "62": 9.97438, + "63": 9.36298, + "64": 9.76793, + "65": 8.93913, + "66": 9.68918, + "67": 9.36638, + "68": 9.77507, + "69": 9.78344, + "70": 9.72196, + "71": 9.60806, + "72": 9.57714, + "73": 9.48934, + "74": 8.94008, + "75": 9.40867, + "76": 9.08075, + "77": 10.05717, + "78": 9.72281, + "79": 9.36465, + "80": 9.39746, + "81": 9.47553, + "82": 9.6886, + "83": 9.30263, + "84": 9.41008, + "85": 9.60793, + "86": 9.07115, + "87": 9.58676, + "88": 9.74129, + "89": 9.5986, + "90": 9.81041, + "91": 9.33113, + "92": 9.35502, + "93": 9.07481, + "94": 8.82745, + "95": 9.51149, + "96": 9.51876, + "97": 9.30173, + "98": 9.66726, + "99": 8.88087, + "100": 9.39727 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1624.0, + "2": 1758.0, + "3": 1678.0, + "4": 1680.0, + "5": 1824.0, + "6": 1899.0, + "7": 1871.0, + "8": 1677.0, + "9": 1802.0, + "10": 1414.0, + "11": 1904.0, + "12": 1630.0, + "13": 1956.0, + "14": 1777.0, + "15": 1902.0, + "16": 1801.0, + "17": 1830.0, + "18": 1630.0, + "19": 1835.0, + "20": 1642.0, + "21": 1752.0, + "22": 1696.0, + "23": 2085.0, + "24": 1618.0, + "25": 1593.0, + "26": 1722.0, + "27": 1778.0, + "28": 2042.0, + "29": 1900.0, + "30": 2001.0, + "31": 1592.0, + "32": 1757.0, + "33": 2116.0, + "34": 1924.0, + "35": 1904.0, + "36": 1852.0, + "37": 2382.0, + "38": 2195.0, + "39": 2267.0, + "40": 2335.0, + "41": 2223.0, + "42": 2317.0, + "43": 2069.0, + "44": 2060.0, + "45": 2140.0, + "46": 2397.0, + "47": 2464.0, + "48": 2455.0, + "49": 2276.0, + "50": 2374.0, + "51": 2574.0, + "52": 2457.0, + "53": 2905.0, + "54": 2609.0, + "55": 2220.0, + "56": 2663.0, + "57": 2258.0, + "58": 2898.0, + "59": 2676.0, + "60": 2397.0, + "61": 3048.0, + "62": 2533.0, + "63": 2370.0, + "64": 2975.0, + "65": 2591.0, + "66": 3065.0, + "67": 2732.0, + "68": 2870.0, + "69": 2955.0, + "70": 3112.0, + "71": 2989.0, + "72": 2451.0, + "73": 2881.0, + "74": 1859.0, + "75": 2649.0, + "76": 3026.0, + "77": 3316.0, + "78": 3212.0, + "79": 3183.0, + "80": 3262.0, + "81": 3669.0, + "82": 3187.0, + "83": 2798.0, + "84": 3209.0, + "85": 3309.0, + "86": 2738.0, + "87": 3804.0, + "88": 2989.0, + "89": 3327.0, + "90": 3031.0, + "91": 2720.0, + "92": 2972.0, + "93": 2719.0, + "94": 3387.0, + "95": 3321.0, + "96": 3342.0, + "97": 3191.0, + "98": 3533.0, + "99": 3214.0, + "100": 3318.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 716834304.0, + "2": 716834304.0, + "3": 716834304.0, + "4": 716834304.0, + "5": 716834304.0, + "6": 716834304.0, + "7": 716834304.0, + "8": 716834304.0, + "9": 716834304.0, + "10": 716834304.0, + "11": 716834304.0, + "12": 716834304.0, + "13": 716834304.0, + "14": 716834304.0, + "15": 716834304.0, + "16": 716834304.0, + "17": 716834304.0, + "18": 716834304.0, + "19": 716834304.0, + "20": 716834304.0, + "21": 716834304.0, + "22": 716834304.0, + "23": 716834304.0, + "24": 716834304.0, + "25": 716834304.0, + "26": 716834304.0, + "27": 716834304.0, + "28": 716834304.0, + "29": 716834304.0, + "30": 716834304.0, + "31": 716834304.0, + "32": 716834304.0, + "33": 716834304.0, + "34": 716834304.0, + "35": 716834304.0, + "36": 716834304.0, + "37": 716834304.0, + "38": 716834304.0, + "39": 716834304.0, + "40": 716834304.0, + "41": 716834304.0, + "42": 716834304.0, + "43": 716834304.0, + "44": 716834304.0, + "45": 716834304.0, + "46": 716834304.0, + "47": 716834304.0, + "48": 716834304.0, + "49": 716834304.0, + "50": 716834304.0, + "51": 716834304.0, + "52": 716834304.0, + "53": 716834304.0, + "54": 716834304.0, + "55": 716834304.0, + "56": 716834304.0, + "57": 716834304.0, + "58": 716834304.0, + "59": 716834304.0, + "60": 716834304.0, + "61": 716834304.0, + "62": 716834304.0, + "63": 716834304.0, + "64": 716834304.0, + "65": 716834304.0, + "66": 716834304.0, + "67": 716834304.0, + "68": 716834304.0, + "69": 716834304.0, + "70": 716834304.0, + "71": 716834304.0, + "72": 716834304.0, + "73": 716834304.0, + "74": 716834304.0, + "75": 716834304.0, + "76": 716834304.0, + "77": 716834304.0, + "78": 716834304.0, + "79": 716834304.0, + "80": 716834304.0, + "81": 716834304.0, + "82": 716834304.0, + "83": 716834304.0, + "84": 716834304.0, + "85": 716834304.0, + "86": 716834304.0, + "87": 716834304.0, + "88": 716834304.0, + "89": 716834304.0, + "90": 716834304.0, + "91": 716834304.0, + "92": 716834304.0, + "93": 716834304.0, + "94": 716834304.0, + "95": 716834304.0, + "96": 716834304.0, + "97": 716834304.0, + "98": 716834304.0, + "99": 716834304.0, + "100": 716834304.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910424576.0, + "2": 2193074176.0, + "3": 2193074176.0, + "4": 2193074176.0, + "5": 2193074176.0, + "6": 2193074176.0, + "7": 2193074176.0, + "8": 2193074176.0, + "9": 2193074176.0, + "10": 2193074176.0, + "11": 2193074176.0, + "12": 2193074176.0, + "13": 2193074176.0, + "14": 2193074176.0, + "15": 2193074176.0, + "16": 2193074176.0, + "17": 2193074176.0, + "18": 2193074176.0, + "19": 2193074176.0, + "20": 2193074176.0, + "21": 2193074176.0, + "22": 2193074176.0, + "23": 2193074176.0, + "24": 2193074176.0, + "25": 2193074176.0, + "26": 2193074176.0, + "27": 2193074176.0, + "28": 2193074176.0, + "29": 2193074176.0, + "30": 2193074176.0, + "31": 2193074176.0, + "32": 2193074176.0, + "33": 2193074176.0, + "34": 2193074176.0, + "35": 2193074176.0, + "36": 2193074176.0, + "37": 2193074176.0, + "38": 2193074176.0, + "39": 2193074176.0, + "40": 2193074176.0, + "41": 2193074176.0, + "42": 2193074176.0, + "43": 2193074176.0, + "44": 2193074176.0, + "45": 2193074176.0, + "46": 2193074176.0, + "47": 2193074176.0, + "48": 2193074176.0, + "49": 2193074176.0, + "50": 2193074176.0, + "51": 2193074176.0, + "52": 2193074176.0, + "53": 2193074176.0, + "54": 2193074176.0, + "55": 2193074176.0, + "56": 2193074176.0, + "57": 2193074176.0, + "58": 2193074176.0, + "59": 2193074176.0, + "60": 2193074176.0, + "61": 2193074176.0, + "62": 2193074176.0, + "63": 2193074176.0, + "64": 2193074176.0, + "65": 2193074176.0, + "66": 2193074176.0, + "67": 2193074176.0, + "68": 2193074176.0, + "69": 2193074176.0, + "70": 2193074176.0, + "71": 2193074176.0, + "72": 2193074176.0, + "73": 2193074176.0, + "74": 2193074176.0, + "75": 2193074176.0, + "76": 2193074176.0, + "77": 2193074176.0, + "78": 2193074176.0, + "79": 2193074176.0, + "80": 2193074176.0, + "81": 2193074176.0, + "82": 2193074176.0, + "83": 2193074176.0, + "84": 2193074176.0, + "85": 2193074176.0, + "86": 2193074176.0, + "87": 2193074176.0, + "88": 2193074176.0, + "89": 2193074176.0, + "90": 2193074176.0, + "91": 2193074176.0, + "92": 2193074176.0, + "93": 2193074176.0, + "94": 2193074176.0, + "95": 2193074176.0, + "96": 2193074176.0, + "97": 2193074176.0, + "98": 2193074176.0, + "99": 2193074176.0, + "100": 2193074176.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.43574, + "2": 0.18308, + "3": 0.16294, + "4": 0.15632, + "5": 0.15517, + "6": 0.15061, + "7": 0.15109, + "8": 0.1538, + "9": 0.15077, + "10": 0.15142, + "11": 0.15024, + "12": 0.15039, + "13": 0.14987, + "14": 0.15044, + "15": 0.1495, + "16": 0.15003, + "17": 0.14988, + "18": 0.1497, + "19": 0.15459, + "20": 0.15076, + "21": 0.1498, + "22": 0.15044, + "23": 0.15051, + "24": 0.15062, + "25": 0.14953, + "26": 0.15047, + "27": 0.14851, + "28": 0.14802, + "29": 0.14861, + "30": 0.1485, + "31": 0.1498, + "32": 0.14871, + "33": 0.1485, + "34": 0.14707, + "35": 0.14796, + "36": 0.14719, + "37": 0.15012, + "38": 0.14804, + "39": 0.1487, + "40": 0.14779, + "41": 0.14844, + "42": 0.1496, + "43": 0.15014, + "44": 0.14977, + "45": 0.1478, + "46": 0.14891, + "47": 0.14844, + "48": 0.1488, + "49": 0.14931, + "50": 0.14761, + "51": 0.15888, + "52": 0.1517, + "53": 0.14904, + "54": 0.17961, + "55": 0.14804, + "56": 0.1496, + "57": 0.1487, + "58": 0.14801, + "59": 0.14729, + "60": 0.14749, + "61": 0.14745, + "62": 0.1471, + "63": 0.14817, + "64": 0.1497, + "65": 0.14753, + "66": 0.14753, + "67": 0.14859, + "68": 0.14714, + "69": 0.14776, + "70": 0.14847, + "71": 0.14829, + "72": 0.14858, + "73": 0.14828, + "74": 0.14783, + "75": 0.14793, + "76": 0.14768, + "77": 0.14752, + "78": 0.14931, + "79": 0.15045, + "80": 0.14813, + "81": 0.1489, + "82": 0.1475, + "83": 0.14844, + "84": 0.1489, + "85": 0.14809, + "86": 0.14835, + "87": 0.14718, + "88": 0.14876, + "89": 0.14859, + "90": 0.1479, + "91": 0.14803, + "92": 0.14798, + "93": 0.14876, + "94": 0.14705, + "95": 0.14837, + "96": 0.14805, + "97": 0.14837, + "98": 0.14721, + "99": 0.14843, + "100": 0.14828 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json new file mode 100644 index 00000000000..801e633e981 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.90314, + "2": 10.88897, + "3": 10.89725, + "4": 10.88248, + "5": 10.89447, + "6": 10.91226, + "7": 10.89775, + "8": 10.88643, + "9": 10.89227, + "10": 10.8849, + "11": 10.91413, + "12": 10.88666, + "13": 10.89283, + "14": 10.90486, + "15": 10.83878, + "16": 10.84927, + "17": 10.83248, + "18": 10.83371, + "19": 10.83359, + "20": 10.74074, + "21": 10.7043, + "22": 10.59893, + "23": 10.72222, + "24": 10.60745, + "25": 10.57467, + "26": 10.62999, + "27": 10.62364, + "28": 10.57229, + "29": 10.6073, + "30": 10.37766, + "31": 10.15362, + "32": 10.47609, + "33": 10.48062, + "34": 10.24216, + "35": 10.29035, + "36": 10.25955, + "37": 10.36145, + "38": 10.21396, + "39": 10.44502, + "40": 10.11492, + "41": 10.1605, + "42": 10.23468, + "43": 9.85032, + "44": 9.97764, + "45": 9.85681, + "46": 9.8307, + "47": 10.17976, + "48": 9.85811, + "49": 9.54378, + "50": 9.93469, + "51": 9.86793, + "52": 9.76274, + "53": 10.10895, + "54": 9.95538, + "55": 9.8756, + "56": 9.64751, + "57": 9.48989, + "58": 9.85502, + "59": 9.59457, + "60": 9.52968, + "61": 9.69589, + "62": 10.01676, + "63": 9.38778, + "64": 9.80211, + "65": 8.95119, + "66": 9.72857, + "67": 9.37577, + "68": 9.80463, + "69": 9.81, + "70": 9.7662, + "71": 9.63135, + "72": 9.5784, + "73": 9.52148, + "74": 8.94976, + "75": 9.43087, + "76": 9.08489, + "77": 10.089, + "78": 9.72754, + "79": 9.37612, + "80": 9.40849, + "81": 9.49766, + "82": 9.71298, + "83": 9.33332, + "84": 9.43928, + "85": 9.63373, + "86": 9.07038, + "87": 9.61245, + "88": 9.78304, + "89": 9.60878, + "90": 9.85164, + "91": 9.34542, + "92": 9.38281, + "93": 9.07319, + "94": 8.81684, + "95": 9.51809, + "96": 9.54033, + "97": 9.34061, + "98": 9.70134, + "99": 8.88786, + "100": 9.43285 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727000.0, + "2": 22925616.0, + "3": 22596924.0, + "4": 23219540.0, + "5": 22714814.0, + "6": 23021786.0, + "7": 22771458.0, + "8": 22926012.0, + "9": 22842856.0, + "10": 22918360.0, + "11": 22500702.0, + "12": 22459866.0, + "13": 22916820.0, + "14": 22389026.0, + "15": 22821260.0, + "16": 22830812.0, + "17": 22818944.0, + "18": 22582240.0, + "19": 22618380.0, + "20": 22694352.0, + "21": 22740180.0, + "22": 22800024.0, + "23": 22540132.0, + "24": 22771492.0, + "25": 22818970.0, + "26": 22546852.0, + "27": 22468504.0, + "28": 22453052.0, + "29": 22529222.0, + "30": 22631432.0, + "31": 22955696.0, + "32": 22585238.0, + "33": 22557676.0, + "34": 22835412.0, + "35": 22788032.0, + "36": 22589678.0, + "37": 22497140.0, + "38": 22896132.0, + "39": 22801314.0, + "40": 22658064.0, + "41": 22659700.0, + "42": 22667816.0, + "43": 22976356.0, + "44": 22746708.0, + "45": 22675272.0, + "46": 22884382.0, + "47": 22634556.0, + "48": 22928080.0, + "49": 22727538.0, + "50": 22905284.0, + "51": 22791326.0, + "52": 22749392.0, + "53": 22925970.0, + "54": 22839434.0, + "55": 22518416.0, + "56": 22877660.0, + "57": 23113304.0, + "58": 22845008.0, + "59": 22715512.0, + "60": 22743058.0, + "61": 22723950.0, + "62": 22673248.0, + "63": 22846074.0, + "64": 22823228.0, + "65": 23060212.0, + "66": 22729902.0, + "67": 22907278.0, + "68": 22610092.0, + "69": 22584360.0, + "70": 22829348.0, + "71": 22749420.0, + "72": 22655446.0, + "73": 22740974.0, + "74": 23048296.0, + "75": 23053922.0, + "76": 22901008.0, + "77": 22272806.0, + "78": 22789370.0, + "79": 22743288.0, + "80": 22706236.0, + "81": 22890976.0, + "82": 22777092.0, + "83": 22839240.0, + "84": 23010352.0, + "85": 22712004.0, + "86": 23103740.0, + "87": 22734788.0, + "88": 22637620.0, + "89": 22499200.0, + "90": 22972420.0, + "91": 22766428.0, + "92": 22808890.0, + "93": 22659888.0, + "94": 22910970.0, + "95": 23048514.0, + "96": 22829470.0, + "97": 22608826.0, + "98": 22763528.0, + "99": 22905754.0, + "100": 23016268.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 719180288.0, + "2": 719180288.0, + "3": 719180288.0, + "4": 719180288.0, + "5": 719180288.0, + "6": 719180288.0, + "7": 719180288.0, + "8": 719180288.0, + "9": 719180288.0, + "10": 719180288.0, + "11": 719180288.0, + "12": 719180288.0, + "13": 719180288.0, + "14": 719180288.0, + "15": 719180288.0, + "16": 719180288.0, + "17": 719180288.0, + "18": 719180288.0, + "19": 719180288.0, + "20": 719180288.0, + "21": 719180288.0, + "22": 719180288.0, + "23": 719180288.0, + "24": 719180288.0, + "25": 719180288.0, + "26": 719180288.0, + "27": 719180288.0, + "28": 719180288.0, + "29": 719180288.0, + "30": 719180288.0, + "31": 719180288.0, + "32": 719180288.0, + "33": 719180288.0, + "34": 719180288.0, + "35": 719180288.0, + "36": 719180288.0, + "37": 719180288.0, + "38": 719180288.0, + "39": 719180288.0, + "40": 719180288.0, + "41": 719180288.0, + "42": 719180288.0, + "43": 719180288.0, + "44": 719180288.0, + "45": 719180288.0, + "46": 719180288.0, + "47": 719180288.0, + "48": 719180288.0, + "49": 719180288.0, + "50": 719180288.0, + "51": 719180288.0, + "52": 719180288.0, + "53": 719180288.0, + "54": 719180288.0, + "55": 719180288.0, + "56": 719180288.0, + "57": 719180288.0, + "58": 719180288.0, + "59": 719180288.0, + "60": 719180288.0, + "61": 719180288.0, + "62": 719180288.0, + "63": 719180288.0, + "64": 719180288.0, + "65": 719180288.0, + "66": 719180288.0, + "67": 719180288.0, + "68": 719180288.0, + "69": 719180288.0, + "70": 719180288.0, + "71": 719180288.0, + "72": 719180288.0, + "73": 719180288.0, + "74": 719180288.0, + "75": 719180288.0, + "76": 719180288.0, + "77": 719180288.0, + "78": 719180288.0, + "79": 719180288.0, + "80": 719180288.0, + "81": 719180288.0, + "82": 719180288.0, + "83": 719180288.0, + "84": 719180288.0, + "85": 719180288.0, + "86": 719180288.0, + "87": 719180288.0, + "88": 719180288.0, + "89": 719180288.0, + "90": 719180288.0, + "91": 719180288.0, + "92": 719180288.0, + "93": 719180288.0, + "94": 719180288.0, + "95": 719180288.0, + "96": 719180288.0, + "97": 719180288.0, + "98": 719180288.0, + "99": 719180288.0, + "100": 719180288.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910562816.0, + "2": 2195420160.0, + "3": 2195420160.0, + "4": 2195420160.0, + "5": 2195420160.0, + "6": 2195420160.0, + "7": 2195420160.0, + "8": 2195420160.0, + "9": 2195420160.0, + "10": 2195420160.0, + "11": 2195420160.0, + "12": 2195420160.0, + "13": 2195420160.0, + "14": 2195420160.0, + "15": 2195420160.0, + "16": 2195420160.0, + "17": 2195420160.0, + "18": 2195420160.0, + "19": 2195420160.0, + "20": 2195420160.0, + "21": 2195420160.0, + "22": 2195420160.0, + "23": 2195420160.0, + "24": 2195420160.0, + "25": 2195420160.0, + "26": 2195420160.0, + "27": 2195420160.0, + "28": 2195420160.0, + "29": 2195420160.0, + "30": 2195420160.0, + "31": 2195420160.0, + "32": 2195420160.0, + "33": 2195420160.0, + "34": 2195420160.0, + "35": 2195420160.0, + "36": 2195420160.0, + "37": 2195420160.0, + "38": 2195420160.0, + "39": 2195420160.0, + "40": 2195420160.0, + "41": 2195420160.0, + "42": 2195420160.0, + "43": 2195420160.0, + "44": 2195420160.0, + "45": 2195420160.0, + "46": 2195420160.0, + "47": 2195420160.0, + "48": 2195420160.0, + "49": 2195420160.0, + "50": 2195420160.0, + "51": 2195420160.0, + "52": 2195420160.0, + "53": 2195420160.0, + "54": 2195420160.0, + "55": 2195420160.0, + "56": 2195420160.0, + "57": 2195420160.0, + "58": 2195420160.0, + "59": 2195420160.0, + "60": 2195420160.0, + "61": 2195420160.0, + "62": 2195420160.0, + "63": 2195420160.0, + "64": 2195420160.0, + "65": 2195420160.0, + "66": 2195420160.0, + "67": 2195420160.0, + "68": 2195420160.0, + "69": 2195420160.0, + "70": 2195420160.0, + "71": 2195420160.0, + "72": 2195420160.0, + "73": 2195420160.0, + "74": 2195420160.0, + "75": 2195420160.0, + "76": 2195420160.0, + "77": 2195420160.0, + "78": 2195420160.0, + "79": 2195420160.0, + "80": 2195420160.0, + "81": 2195420160.0, + "82": 2195420160.0, + "83": 2195420160.0, + "84": 2195420160.0, + "85": 2195420160.0, + "86": 2195420160.0, + "87": 2195420160.0, + "88": 2195420160.0, + "89": 2195420160.0, + "90": 2195420160.0, + "91": 2195420160.0, + "92": 2195420160.0, + "93": 2195420160.0, + "94": 2195420160.0, + "95": 2195420160.0, + "96": 2195420160.0, + "97": 2195420160.0, + "98": 2195420160.0, + "99": 2195420160.0, + "100": 2195420160.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.03051, + "2": 0.1884, + "3": 0.27987, + "4": 0.15687, + "5": 0.15614, + "6": 0.1563, + "7": 0.15622, + "8": 0.1563, + "9": 0.15473, + "10": 0.15615, + "11": 0.15528, + "12": 0.15754, + "13": 0.15661, + "14": 0.15677, + "15": 0.15572, + "16": 0.15582, + "17": 0.1571, + "18": 0.15664, + "19": 0.1556, + "20": 0.15592, + "21": 0.15572, + "22": 0.15642, + "23": 0.15643, + "24": 0.15743, + "25": 0.15666, + "26": 0.15552, + "27": 0.15475, + "28": 0.15586, + "29": 0.15603, + "30": 0.15463, + "31": 0.15712, + "32": 0.15442, + "33": 0.15543, + "34": 0.1557, + "35": 0.15682, + "36": 0.15539, + "37": 0.1553, + "38": 0.15578, + "39": 0.15667, + "40": 0.15715, + "41": 0.15704, + "42": 0.1566, + "43": 0.15655, + "44": 0.15629, + "45": 0.15584, + "46": 0.15734, + "47": 0.15735, + "48": 0.15572, + "49": 0.15706, + "50": 0.15561, + "51": 0.16957, + "52": 0.1587, + "53": 0.16014, + "54": 0.15805, + "55": 0.1578, + "56": 0.15801, + "57": 0.15813, + "58": 0.1574, + "59": 0.15781, + "60": 0.15923, + "61": 0.15655, + "62": 0.15633, + "63": 0.15583, + "64": 0.15734, + "65": 0.15761, + "66": 0.15822, + "67": 0.15755, + "68": 0.15815, + "69": 0.15816, + "70": 0.15813, + "71": 0.15747, + "72": 0.1574, + "73": 0.15783, + "74": 0.15766, + "75": 0.15527, + "76": 0.15579, + "77": 0.15483, + "78": 0.15482, + "79": 0.15404, + "80": 0.15431, + "81": 0.154, + "82": 0.15512, + "83": 0.15513, + "84": 0.15371, + "85": 0.15488, + "86": 0.15465, + "87": 0.15412, + "88": 0.15403, + "89": 0.15487, + "90": 0.15518, + "91": 0.15549, + "92": 0.154, + "93": 0.15405, + "94": 0.15438, + "95": 0.15444, + "96": 0.1534, + "97": 0.15487, + "98": 0.15398, + "99": 0.15434, + "100": 0.15391 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json index 11578ee6c3e..307cec2659c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.90105, - "5": 10.89686, - "10": 10.88269, - "15": 10.83975, - "20": 10.74037, - "25": 10.57931, - "30": 10.37738, - "35": 10.29033, - "40": 10.1156, - "45": 9.85639, - "50": 9.93378, - "55": 9.87553, - "60": 9.528, - "65": 8.95041, - "70": 9.76634, - "75": 9.4304, - "80": 9.40916, - "85": 9.63365, - "90": 9.8516, - "95": 9.51874, - "100": 9.43257 + "1": 10.90314, + "2": 10.88897, + "3": 10.89725, + "4": 10.88248, + "5": 10.89447, + "6": 10.91226, + "7": 10.89775, + "8": 10.88643, + "9": 10.89227, + "10": 10.8849, + "11": 10.91413, + "12": 10.88666, + "13": 10.89283, + "14": 10.90486, + "15": 10.83878, + "16": 10.84927, + "17": 10.83248, + "18": 10.83371, + "19": 10.83359, + "20": 10.74074, + "21": 10.7043, + "22": 10.59893, + "23": 10.72222, + "24": 10.60745, + "25": 10.57467, + "26": 10.62999, + "27": 10.62364, + "28": 10.57229, + "29": 10.6073, + "30": 10.37766, + "31": 10.15362, + "32": 10.47609, + "33": 10.48062, + "34": 10.24216, + "35": 10.29035, + "36": 10.25955, + "37": 10.36145, + "38": 10.21396, + "39": 10.44502, + "40": 10.11492, + "41": 10.1605, + "42": 10.23468, + "43": 9.85032, + "44": 9.97764, + "45": 9.85681, + "46": 9.8307, + "47": 10.17976, + "48": 9.85811, + "49": 9.54378, + "50": 9.93469, + "51": 9.86793, + "52": 9.76274, + "53": 10.10895, + "54": 9.95538, + "55": 9.8756, + "56": 9.64751, + "57": 9.48989, + "58": 9.85502, + "59": 9.59457, + "60": 9.52968, + "61": 9.69589, + "62": 10.01676, + "63": 9.38778, + "64": 9.80211, + "65": 8.95119, + "66": 9.72857, + "67": 9.37577, + "68": 9.80463, + "69": 9.81, + "70": 9.7662, + "71": 9.63135, + "72": 9.5784, + "73": 9.52148, + "74": 8.94976, + "75": 9.43087, + "76": 9.08489, + "77": 10.089, + "78": 9.72754, + "79": 9.37612, + "80": 9.40849, + "81": 9.49766, + "82": 9.71298, + "83": 9.33332, + "84": 9.43928, + "85": 9.63373, + "86": 9.07038, + "87": 9.61245, + "88": 9.78304, + "89": 9.60878, + "90": 9.85164, + "91": 9.34542, + "92": 9.38281, + "93": 9.07319, + "94": 8.81684, + "95": 9.51809, + "96": 9.54033, + "97": 9.34061, + "98": 9.70134, + "99": 8.88786, + "100": 9.43285 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 22727086.0, - "5": 22714736.0, - "10": 22918912.0, - "15": 22821334.0, - "20": 22694008.0, - "25": 22818912.0, - "30": 22631266.0, - "35": 22787944.0, - "40": 22658196.0, - "45": 22675296.0, - "50": 22904808.0, - "55": 22518754.0, - "60": 22743504.0, - "65": 23060460.0, - "70": 22829772.0, - "75": 23054048.0, - "80": 22706308.0, - "85": 22712054.0, - "90": 22972128.0, - "95": 23048144.0, - "100": 23015824.0 + "1": 22727000.0, + "2": 22925616.0, + "3": 22596924.0, + "4": 23219540.0, + "5": 22714814.0, + "6": 23021786.0, + "7": 22771458.0, + "8": 22926012.0, + "9": 22842856.0, + "10": 22918360.0, + "11": 22500702.0, + "12": 22459866.0, + "13": 22916820.0, + "14": 22389026.0, + "15": 22821260.0, + "16": 22830812.0, + "17": 22818944.0, + "18": 22582240.0, + "19": 22618380.0, + "20": 22694352.0, + "21": 22740180.0, + "22": 22800024.0, + "23": 22540132.0, + "24": 22771492.0, + "25": 22818970.0, + "26": 22546852.0, + "27": 22468504.0, + "28": 22453052.0, + "29": 22529222.0, + "30": 22631432.0, + "31": 22955696.0, + "32": 22585238.0, + "33": 22557676.0, + "34": 22835412.0, + "35": 22788032.0, + "36": 22589678.0, + "37": 22497140.0, + "38": 22896132.0, + "39": 22801314.0, + "40": 22658064.0, + "41": 22659700.0, + "42": 22667816.0, + "43": 22976356.0, + "44": 22746708.0, + "45": 22675272.0, + "46": 22884382.0, + "47": 22634556.0, + "48": 22928080.0, + "49": 22727538.0, + "50": 22905284.0, + "51": 22791326.0, + "52": 22749392.0, + "53": 22925970.0, + "54": 22839434.0, + "55": 22518416.0, + "56": 22877660.0, + "57": 23113304.0, + "58": 22845008.0, + "59": 22715512.0, + "60": 22743058.0, + "61": 22723950.0, + "62": 22673248.0, + "63": 22846074.0, + "64": 22823228.0, + "65": 23060212.0, + "66": 22729902.0, + "67": 22907278.0, + "68": 22610092.0, + "69": 22584360.0, + "70": 22829348.0, + "71": 22749420.0, + "72": 22655446.0, + "73": 22740974.0, + "74": 23048296.0, + "75": 23053922.0, + "76": 22901008.0, + "77": 22272806.0, + "78": 22789370.0, + "79": 22743288.0, + "80": 22706236.0, + "81": 22890976.0, + "82": 22777092.0, + "83": 22839240.0, + "84": 23010352.0, + "85": 22712004.0, + "86": 23103740.0, + "87": 22734788.0, + "88": 22637620.0, + "89": 22499200.0, + "90": 22972420.0, + "91": 22766428.0, + "92": 22808890.0, + "93": 22659888.0, + "94": 22910970.0, + "95": 23048514.0, + "96": 22829470.0, + "97": 22608826.0, + "98": 22763528.0, + "99": 22905754.0, + "100": 23016268.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 717082624.0, - "5": 717082624.0, - "10": 717082624.0, - "15": 717082624.0, - "20": 717082624.0, - "25": 717082624.0, - "30": 717082624.0, - "35": 717082624.0, - "40": 717082624.0, - "45": 717082624.0, - "50": 717082624.0, - "55": 717082624.0, - "60": 717082624.0, - "65": 717082624.0, - "70": 717082624.0, - "75": 717082624.0, - "80": 717082624.0, - "85": 717082624.0, - "90": 717082624.0, - "95": 717082624.0, - "100": 717082624.0 + "1": 719180288.0, + "2": 719180288.0, + "3": 719180288.0, + "4": 719180288.0, + "5": 719180288.0, + "6": 719180288.0, + "7": 719180288.0, + "8": 719180288.0, + "9": 719180288.0, + "10": 719180288.0, + "11": 719180288.0, + "12": 719180288.0, + "13": 719180288.0, + "14": 719180288.0, + "15": 719180288.0, + "16": 719180288.0, + "17": 719180288.0, + "18": 719180288.0, + "19": 719180288.0, + "20": 719180288.0, + "21": 719180288.0, + "22": 719180288.0, + "23": 719180288.0, + "24": 719180288.0, + "25": 719180288.0, + "26": 719180288.0, + "27": 719180288.0, + "28": 719180288.0, + "29": 719180288.0, + "30": 719180288.0, + "31": 719180288.0, + "32": 719180288.0, + "33": 719180288.0, + "34": 719180288.0, + "35": 719180288.0, + "36": 719180288.0, + "37": 719180288.0, + "38": 719180288.0, + "39": 719180288.0, + "40": 719180288.0, + "41": 719180288.0, + "42": 719180288.0, + "43": 719180288.0, + "44": 719180288.0, + "45": 719180288.0, + "46": 719180288.0, + "47": 719180288.0, + "48": 719180288.0, + "49": 719180288.0, + "50": 719180288.0, + "51": 719180288.0, + "52": 719180288.0, + "53": 719180288.0, + "54": 719180288.0, + "55": 719180288.0, + "56": 719180288.0, + "57": 719180288.0, + "58": 719180288.0, + "59": 719180288.0, + "60": 719180288.0, + "61": 719180288.0, + "62": 719180288.0, + "63": 719180288.0, + "64": 719180288.0, + "65": 719180288.0, + "66": 719180288.0, + "67": 719180288.0, + "68": 719180288.0, + "69": 719180288.0, + "70": 719180288.0, + "71": 719180288.0, + "72": 719180288.0, + "73": 719180288.0, + "74": 719180288.0, + "75": 719180288.0, + "76": 719180288.0, + "77": 719180288.0, + "78": 719180288.0, + "79": 719180288.0, + "80": 719180288.0, + "81": 719180288.0, + "82": 719180288.0, + "83": 719180288.0, + "84": 719180288.0, + "85": 719180288.0, + "86": 719180288.0, + "87": 719180288.0, + "88": 719180288.0, + "89": 719180288.0, + "90": 719180288.0, + "91": 719180288.0, + "92": 719180288.0, + "93": 719180288.0, + "94": 719180288.0, + "95": 719180288.0, + "96": 719180288.0, + "97": 719180288.0, + "98": 719180288.0, + "99": 719180288.0, + "100": 719180288.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 2399852544.0, - "5": 2683661312.0, - "10": 2683661312.0, - "15": 2683661312.0, - "20": 2683661312.0, - "25": 2683661312.0, - "30": 2683661312.0, - "35": 2683661312.0, - "40": 2683661312.0, - "45": 2683661312.0, - "50": 2683661312.0, - "55": 2683661312.0, - "60": 2683661312.0, - "65": 2683661312.0, - "70": 2683661312.0, - "75": 2683661312.0, - "80": 2683661312.0, - "85": 2683661312.0, - "90": 2683661312.0, - "95": 2683661312.0, - "100": 2683661312.0 + "1": 1910562816.0, + "2": 2195420160.0, + "3": 2195420160.0, + "4": 2195420160.0, + "5": 2195420160.0, + "6": 2195420160.0, + "7": 2195420160.0, + "8": 2195420160.0, + "9": 2195420160.0, + "10": 2195420160.0, + "11": 2195420160.0, + "12": 2195420160.0, + "13": 2195420160.0, + "14": 2195420160.0, + "15": 2195420160.0, + "16": 2195420160.0, + "17": 2195420160.0, + "18": 2195420160.0, + "19": 2195420160.0, + "20": 2195420160.0, + "21": 2195420160.0, + "22": 2195420160.0, + "23": 2195420160.0, + "24": 2195420160.0, + "25": 2195420160.0, + "26": 2195420160.0, + "27": 2195420160.0, + "28": 2195420160.0, + "29": 2195420160.0, + "30": 2195420160.0, + "31": 2195420160.0, + "32": 2195420160.0, + "33": 2195420160.0, + "34": 2195420160.0, + "35": 2195420160.0, + "36": 2195420160.0, + "37": 2195420160.0, + "38": 2195420160.0, + "39": 2195420160.0, + "40": 2195420160.0, + "41": 2195420160.0, + "42": 2195420160.0, + "43": 2195420160.0, + "44": 2195420160.0, + "45": 2195420160.0, + "46": 2195420160.0, + "47": 2195420160.0, + "48": 2195420160.0, + "49": 2195420160.0, + "50": 2195420160.0, + "51": 2195420160.0, + "52": 2195420160.0, + "53": 2195420160.0, + "54": 2195420160.0, + "55": 2195420160.0, + "56": 2195420160.0, + "57": 2195420160.0, + "58": 2195420160.0, + "59": 2195420160.0, + "60": 2195420160.0, + "61": 2195420160.0, + "62": 2195420160.0, + "63": 2195420160.0, + "64": 2195420160.0, + "65": 2195420160.0, + "66": 2195420160.0, + "67": 2195420160.0, + "68": 2195420160.0, + "69": 2195420160.0, + "70": 2195420160.0, + "71": 2195420160.0, + "72": 2195420160.0, + "73": 2195420160.0, + "74": 2195420160.0, + "75": 2195420160.0, + "76": 2195420160.0, + "77": 2195420160.0, + "78": 2195420160.0, + "79": 2195420160.0, + "80": 2195420160.0, + "81": 2195420160.0, + "82": 2195420160.0, + "83": 2195420160.0, + "84": 2195420160.0, + "85": 2195420160.0, + "86": 2195420160.0, + "87": 2195420160.0, + "88": 2195420160.0, + "89": 2195420160.0, + "90": 2195420160.0, + "91": 2195420160.0, + "92": 2195420160.0, + "93": 2195420160.0, + "94": 2195420160.0, + "95": 2195420160.0, + "96": 2195420160.0, + "97": 2195420160.0, + "98": 2195420160.0, + "99": 2195420160.0, + "100": 2195420160.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 18.12809, - "5": 0.18057, - "10": 0.17706, - "15": 0.17756, - "20": 0.17731, - "25": 0.17728, - "30": 0.1768, - "35": 0.17506, - "40": 0.1753, - "45": 0.17703, - "50": 0.17481, - "55": 0.17327, - "60": 0.17184, - "65": 0.17221, - "70": 0.17198, - "75": 0.1719, - "80": 0.17197, - "85": 0.1726, - "90": 0.17157, - "95": 0.17135, - "100": 0.17214 + "1": 10.1537, + "2": 0.18498, + "3": 0.16024, + "4": 0.16059, + "5": 0.16002, + "6": 0.16103, + "7": 0.1591, + "8": 0.15912, + "9": 0.15909, + "10": 0.1574, + "11": 0.15721, + "12": 0.15764, + "13": 0.16009, + "14": 0.16035, + "15": 0.15973, + "16": 0.15641, + "17": 0.15673, + "18": 0.1565, + "19": 0.15684, + "20": 0.15713, + "21": 0.15762, + "22": 0.15859, + "23": 0.15877, + "24": 0.15973, + "25": 0.15946, + "26": 0.15909, + "27": 0.15855, + "28": 0.15876, + "29": 0.15921, + "30": 0.16148, + "31": 0.15991, + "32": 0.1576, + "33": 0.15829, + "34": 0.15886, + "35": 0.15948, + "36": 0.15819, + "37": 0.15886, + "38": 0.15896, + "39": 0.16029, + "40": 0.15802, + "41": 0.16038, + "42": 0.15965, + "43": 0.15985, + "44": 0.15882, + "45": 0.16056, + "46": 0.1592, + "47": 0.20747, + "48": 0.16124, + "49": 0.16012, + "50": 0.15759, + "51": 0.16615, + "52": 0.15685, + "53": 0.15965, + "54": 0.15787, + "55": 0.15762, + "56": 0.15748, + "57": 0.15807, + "58": 0.15831, + "59": 0.15671, + "60": 0.15765, + "61": 0.15997, + "62": 0.15756, + "63": 0.15822, + "64": 0.15898, + "65": 0.15778, + "66": 0.15853, + "67": 0.15855, + "68": 0.15784, + "69": 0.15777, + "70": 0.15791, + "71": 0.15907, + "72": 0.15986, + "73": 0.15727, + "74": 0.15842, + "75": 0.15738, + "76": 0.15786, + "77": 0.15749, + "78": 0.15761, + "79": 0.15838, + "80": 0.15955, + "81": 0.15796, + "82": 0.15816, + "83": 0.15953, + "84": 0.15849, + "85": 0.15905, + "86": 0.15852, + "87": 0.15827, + "88": 0.15773, + "89": 0.15778, + "90": 0.15679, + "91": 0.1583, + "92": 0.15749, + "93": 0.15843, + "94": 0.15878, + "95": 0.15805, + "96": 0.1588, + "97": 0.15983, + "98": 0.16098, + "99": 0.16131, + "100": 0.15935 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json new file mode 100644 index 00000000000..eee2d1c3d29 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79513, + "16": 10.7958, + "17": 10.76832, + "18": 10.8109, + "19": 10.79887, + "20": 10.69131, + "21": 10.6801, + "22": 10.52152, + "23": 10.7071, + "24": 10.57678, + "25": 10.52316, + "26": 10.59563, + "27": 10.58607, + "28": 10.56175, + "29": 10.56945, + "30": 10.34623, + "31": 10.10035, + "32": 10.45432, + "33": 10.44591, + "34": 10.2072, + "35": 10.26185, + "36": 10.21228, + "37": 10.32449, + "38": 10.16803, + "39": 10.38353, + "40": 10.07219, + "41": 10.13754, + "42": 10.19755, + "43": 9.81134, + "44": 9.93287, + "45": 9.80998, + "46": 9.80859, + "47": 10.12583, + "48": 9.82132, + "49": 9.50738, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1894.0, + "16": 1731.0, + "17": 1848.0, + "18": 1705.0, + "19": 1686.0, + "20": 1620.0, + "21": 1840.0, + "22": 1764.0, + "23": 1937.0, + "24": 1620.0, + "25": 1728.0, + "26": 1727.0, + "27": 1821.0, + "28": 2042.0, + "29": 2029.0, + "30": 1825.0, + "31": 1594.0, + "32": 1903.0, + "33": 2041.0, + "34": 1895.0, + "35": 1908.0, + "36": 1906.0, + "37": 2224.0, + "38": 2150.0, + "39": 2327.0, + "40": 2074.0, + "41": 2314.0, + "42": 2230.0, + "43": 1920.0, + "44": 2115.0, + "45": 1962.0, + "46": 2287.0, + "47": 2481.0, + "48": 2407.0, + "49": 2270.0, + "50": 2349.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3837453824.0, + "2": 4119165440.0, + "3": 4119165440.0, + "4": 4119165440.0, + "5": 4119165440.0, + "6": 4119165440.0, + "7": 4119165440.0, + "8": 4119165440.0, + "9": 4119165440.0, + "10": 4119165440.0, + "11": 4119165440.0, + "12": 4119165440.0, + "13": 4119165440.0, + "14": 4119165440.0, + "15": 4119165440.0, + "16": 4119165440.0, + "17": 4119165440.0, + "18": 4119165440.0, + "19": 4119165440.0, + "20": 4119165440.0, + "21": 4119165440.0, + "22": 4119165440.0, + "23": 4119165440.0, + "24": 4119165440.0, + "25": 4119165440.0, + "26": 4119165440.0, + "27": 4119165440.0, + "28": 4119165440.0, + "29": 4119165440.0, + "30": 4119165440.0, + "31": 4119165440.0, + "32": 4119165440.0, + "33": 4119165440.0, + "34": 4119165440.0, + "35": 4119165440.0, + "36": 4119165440.0, + "37": 4119165440.0, + "38": 4119165440.0, + "39": 4119165440.0, + "40": 4119165440.0, + "41": 4119165440.0, + "42": 4119165440.0, + "43": 4119165440.0, + "44": 4119165440.0, + "45": 4119165440.0, + "46": 4119165440.0, + "47": 4119165440.0, + "48": 4119165440.0, + "49": 4119165440.0, + "50": 4119165440.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.00614, + "2": 0.18804, + "3": 0.28065, + "4": 0.14064, + "5": 0.1388, + "6": 0.13824, + "7": 0.13862, + "8": 0.13843, + "9": 0.14064, + "10": 0.13568, + "11": 0.13353, + "12": 0.13327, + "13": 0.13418, + "14": 0.13368, + "15": 0.13399, + "16": 0.13326, + "17": 0.13409, + "18": 0.13281, + "19": 0.13303, + "20": 0.13395, + "21": 0.13357, + "22": 0.13388, + "23": 0.13403, + "24": 0.1333, + "25": 0.13242, + "26": 0.13302, + "27": 0.134, + "28": 0.13304, + "29": 0.13302, + "30": 0.13398, + "31": 0.13424, + "32": 0.13315, + "33": 0.13365, + "34": 0.13391, + "35": 0.13392, + "36": 0.13316, + "37": 0.13254, + "38": 0.13292, + "39": 0.1333, + "40": 0.13401, + "41": 0.13408, + "42": 0.13349, + "43": 0.13469, + "44": 0.13282, + "45": 0.1344, + "46": 0.13536, + "47": 0.13291, + "48": 0.13374, + "49": 0.13338, + "50": 0.13336 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json index 8faccf70250..4bf73c8b005 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69211, "25": 10.52412, "30": 10.34552, "35": 10.26239, "40": 10.07241, "45": 9.81101, "50": 9.88422}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1601.0, "25": 1635.0, "30": 1936.0, "35": 1908.0, "40": 2100.0, "45": 2098.0, "50": 2333.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 14.9121, "5": 0.1731, "10": 0.17256, "15": 0.1722, "20": 0.17555, "25": 0.17245, "30": 0.17067, "35": 0.17091, "40": 0.17274, "45": 0.17151, "50": 0.17108}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79513, + "16": 10.7958, + "17": 10.76832, + "18": 10.8109, + "19": 10.79887, + "20": 10.69131, + "21": 10.6801, + "22": 10.52152, + "23": 10.7071, + "24": 10.57678, + "25": 10.52316, + "26": 10.59563, + "27": 10.58607, + "28": 10.56175, + "29": 10.56945, + "30": 10.34623, + "31": 10.10035, + "32": 10.45432, + "33": 10.44591, + "34": 10.2072, + "35": 10.26185, + "36": 10.21228, + "37": 10.32449, + "38": 10.16803, + "39": 10.38353, + "40": 10.07219, + "41": 10.13754, + "42": 10.19755, + "43": 9.81134, + "44": 9.93287, + "45": 9.80998, + "46": 9.80859, + "47": 10.12583, + "48": 9.82132, + "49": 9.50738, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1894.0, + "16": 1731.0, + "17": 1848.0, + "18": 1705.0, + "19": 1686.0, + "20": 1620.0, + "21": 1840.0, + "22": 1764.0, + "23": 1937.0, + "24": 1620.0, + "25": 1728.0, + "26": 1727.0, + "27": 1821.0, + "28": 2042.0, + "29": 2029.0, + "30": 1825.0, + "31": 1594.0, + "32": 1903.0, + "33": 2041.0, + "34": 1895.0, + "35": 1908.0, + "36": 1906.0, + "37": 2224.0, + "38": 2150.0, + "39": 2327.0, + "40": 2074.0, + "41": 2314.0, + "42": 2230.0, + "43": 1920.0, + "44": 2115.0, + "45": 1962.0, + "46": 2287.0, + "47": 2481.0, + "48": 2407.0, + "49": 2270.0, + "50": 2349.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3837453824.0, + "2": 4119165440.0, + "3": 4119165440.0, + "4": 4119165440.0, + "5": 4119165440.0, + "6": 4119165440.0, + "7": 4119165440.0, + "8": 4119165440.0, + "9": 4119165440.0, + "10": 4119165440.0, + "11": 4119165440.0, + "12": 4119165440.0, + "13": 4119165440.0, + "14": 4119165440.0, + "15": 4119165440.0, + "16": 4119165440.0, + "17": 4119165440.0, + "18": 4119165440.0, + "19": 4119165440.0, + "20": 4119165440.0, + "21": 4119165440.0, + "22": 4119165440.0, + "23": 4119165440.0, + "24": 4119165440.0, + "25": 4119165440.0, + "26": 4119165440.0, + "27": 4119165440.0, + "28": 4119165440.0, + "29": 4119165440.0, + "30": 4119165440.0, + "31": 4119165440.0, + "32": 4119165440.0, + "33": 4119165440.0, + "34": 4119165440.0, + "35": 4119165440.0, + "36": 4119165440.0, + "37": 4119165440.0, + "38": 4119165440.0, + "39": 4119165440.0, + "40": 4119165440.0, + "41": 4119165440.0, + "42": 4119165440.0, + "43": 4119165440.0, + "44": 4119165440.0, + "45": 4119165440.0, + "46": 4119165440.0, + "47": 4119165440.0, + "48": 4119165440.0, + "49": 4119165440.0, + "50": 4119165440.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.25479, + "2": 0.18004, + "3": 0.15444, + "4": 0.15284, + "5": 0.15391, + "6": 0.14333, + "7": 0.14244, + "8": 0.13997, + "9": 0.14112, + "10": 0.13863, + "11": 0.13707, + "12": 0.13575, + "13": 0.13558, + "14": 0.13535, + "15": 0.13556, + "16": 0.13648, + "17": 0.13495, + "18": 0.1343, + "19": 0.13442, + "20": 0.13441, + "21": 0.1344, + "22": 0.13478, + "23": 0.13473, + "24": 0.13476, + "25": 0.13536, + "26": 0.13345, + "27": 0.1342, + "28": 0.13421, + "29": 0.13479, + "30": 0.13378, + "31": 0.13418, + "32": 0.13411, + "33": 0.13351, + "34": 0.13374, + "35": 0.13406, + "36": 0.13396, + "37": 0.13435, + "38": 0.13356, + "39": 0.13367, + "40": 0.13361, + "41": 0.13454, + "42": 0.13463, + "43": 0.13524, + "44": 0.13356, + "45": 0.13403, + "46": 0.1347, + "47": 0.13379, + "48": 0.1343, + "49": 0.13391, + "50": 0.13371 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..1553ecdd445 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79514, + "16": 10.79576, + "17": 10.76832, + "18": 10.81092, + "19": 10.79889, + "20": 10.69131, + "21": 10.68008, + "22": 10.52146, + "23": 10.70713, + "24": 10.57677, + "25": 10.52315, + "26": 10.59564, + "27": 10.5861, + "28": 10.56176, + "29": 10.56942, + "30": 10.34624, + "31": 10.10032, + "32": 10.45433, + "33": 10.44592, + "34": 10.20725, + "35": 10.26186, + "36": 10.21223, + "37": 10.32453, + "38": 10.16801, + "39": 10.38354, + "40": 10.07222, + "41": 10.13752, + "42": 10.19756, + "43": 9.81134, + "44": 9.93285, + "45": 9.81001, + "46": 9.80858, + "47": 10.12582, + "48": 9.82129, + "49": 9.50739, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1839.0, + "16": 1776.0, + "17": 1750.0, + "18": 1612.0, + "19": 1764.0, + "20": 1649.0, + "21": 1854.0, + "22": 1750.0, + "23": 1909.0, + "24": 1616.0, + "25": 1654.0, + "26": 1755.0, + "27": 1860.0, + "28": 2042.0, + "29": 1953.0, + "30": 1905.0, + "31": 1684.0, + "32": 1831.0, + "33": 2101.0, + "34": 1769.0, + "35": 1915.0, + "36": 1885.0, + "37": 2324.0, + "38": 2169.0, + "39": 2300.0, + "40": 2069.0, + "41": 2353.0, + "42": 2236.0, + "43": 1978.0, + "44": 2022.0, + "45": 2103.0, + "46": 2292.0, + "47": 2413.0, + "48": 2305.0, + "49": 2218.0, + "50": 2321.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 523371008.0, + "2": 523371008.0, + "3": 523371008.0, + "4": 523371008.0, + "5": 523371008.0, + "6": 523371008.0, + "7": 523371008.0, + "8": 523371008.0, + "9": 523371008.0, + "10": 523371008.0, + "11": 523371008.0, + "12": 523371008.0, + "13": 523371008.0, + "14": 523371008.0, + "15": 523371008.0, + "16": 523371008.0, + "17": 523371008.0, + "18": 523371008.0, + "19": 523371008.0, + "20": 523371008.0, + "21": 523371008.0, + "22": 523371008.0, + "23": 523371008.0, + "24": 523371008.0, + "25": 523371008.0, + "26": 523371008.0, + "27": 523371008.0, + "28": 523371008.0, + "29": 523371008.0, + "30": 523371008.0, + "31": 523371008.0, + "32": 523371008.0, + "33": 523371008.0, + "34": 523371008.0, + "35": 523371008.0, + "36": 523371008.0, + "37": 523371008.0, + "38": 523371008.0, + "39": 523371008.0, + "40": 523371008.0, + "41": 523371008.0, + "42": 523371008.0, + "43": 523371008.0, + "44": 523371008.0, + "45": 523371008.0, + "46": 523371008.0, + "47": 523371008.0, + "48": 523371008.0, + "49": 523371008.0, + "50": 523371008.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3768846848.0, + "2": 3913263616.0, + "3": 3913263616.0, + "4": 3913263616.0, + "5": 3913263616.0, + "6": 3913263616.0, + "7": 3913263616.0, + "8": 3913263616.0, + "9": 3913263616.0, + "10": 3913263616.0, + "11": 3913263616.0, + "12": 3913263616.0, + "13": 3913263616.0, + "14": 3913263616.0, + "15": 3913263616.0, + "16": 3913263616.0, + "17": 3913263616.0, + "18": 3913263616.0, + "19": 3913263616.0, + "20": 3913263616.0, + "21": 3913263616.0, + "22": 3913263616.0, + "23": 3913263616.0, + "24": 3913263616.0, + "25": 3913263616.0, + "26": 3913263616.0, + "27": 3913263616.0, + "28": 3913263616.0, + "29": 3913263616.0, + "30": 3913263616.0, + "31": 3913263616.0, + "32": 3913263616.0, + "33": 3913263616.0, + "34": 3913263616.0, + "35": 3913263616.0, + "36": 3913263616.0, + "37": 3913263616.0, + "38": 3913263616.0, + "39": 3913263616.0, + "40": 3913263616.0, + "41": 3913263616.0, + "42": 3913263616.0, + "43": 3913263616.0, + "44": 3913263616.0, + "45": 3913263616.0, + "46": 3913263616.0, + "47": 3913263616.0, + "48": 3913263616.0, + "49": 3913263616.0, + "50": 3913263616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.01582, + "2": 0.17662, + "3": 0.15135, + "4": 0.14999, + "5": 0.14829, + "6": 0.14621, + "7": 0.14816, + "8": 0.14731, + "9": 0.14766, + "10": 0.14515, + "11": 0.30054, + "12": 0.14534, + "13": 0.14429, + "14": 0.14592, + "15": 0.14632, + "16": 0.14618, + "17": 0.14537, + "18": 0.14666, + "19": 0.14384, + "20": 0.14453, + "21": 0.30388, + "22": 0.14466, + "23": 0.14511, + "24": 0.14435, + "25": 0.14401, + "26": 0.14328, + "27": 0.14376, + "28": 0.14434, + "29": 0.14386, + "30": 0.14418, + "31": 0.30313, + "32": 0.14394, + "33": 0.14406, + "34": 0.14377, + "35": 0.14417, + "36": 0.14415, + "37": 0.14393, + "38": 0.14577, + "39": 0.14494, + "40": 0.14489, + "41": 0.30235, + "42": 0.14494, + "43": 0.1472, + "44": 0.14577, + "45": 0.14497, + "46": 0.14619, + "47": 0.14474, + "48": 0.14551, + "49": 0.14554, + "50": 0.14507 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json index dc393d0dffc..35ef87a5085 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.7923, "20": 10.69213, "25": 10.5241, "30": 10.34556, "35": 10.26241, "40": 10.07237, "45": 9.811, "50": 9.88419}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1850.0, "20": 1699.0, "25": 1614.0, "30": 1905.0, "35": 1933.0, "40": 2169.0, "45": 2101.0, "50": 2421.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 523004928.0, "5": 523004928.0, "10": 523004928.0, "15": 523004928.0, "20": 523004928.0, "25": 523004928.0, "30": 523004928.0, "35": 523004928.0, "40": 523004928.0, "45": 523004928.0, "50": 523004928.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3768873984.0, "5": 3912766464.0, "10": 3912766464.0, "15": 3912766464.0, "20": 3912766464.0, "25": 3912766464.0, "30": 3912766464.0, "35": 3912766464.0, "40": 3912766464.0, "45": 3912766464.0, "50": 3912766464.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 18.88705, "5": 0.16956, "10": 0.17448, "15": 0.16853, "20": 0.1715, "25": 0.17071, "30": 0.17343, "35": 0.17213, "40": 0.1719, "45": 0.17357, "50": 0.17228}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79514, + "16": 10.79576, + "17": 10.76832, + "18": 10.81092, + "19": 10.79889, + "20": 10.69131, + "21": 10.68008, + "22": 10.52146, + "23": 10.70713, + "24": 10.57677, + "25": 10.52315, + "26": 10.59564, + "27": 10.5861, + "28": 10.56176, + "29": 10.56942, + "30": 10.34624, + "31": 10.10032, + "32": 10.45433, + "33": 10.44592, + "34": 10.20725, + "35": 10.26186, + "36": 10.21223, + "37": 10.32453, + "38": 10.16801, + "39": 10.38354, + "40": 10.07222, + "41": 10.13752, + "42": 10.19756, + "43": 9.81134, + "44": 9.93285, + "45": 9.81001, + "46": 9.80858, + "47": 10.12582, + "48": 9.82129, + "49": 9.50739, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1839.0, + "16": 1776.0, + "17": 1750.0, + "18": 1612.0, + "19": 1764.0, + "20": 1649.0, + "21": 1854.0, + "22": 1750.0, + "23": 1909.0, + "24": 1616.0, + "25": 1654.0, + "26": 1755.0, + "27": 1860.0, + "28": 2042.0, + "29": 1953.0, + "30": 1905.0, + "31": 1684.0, + "32": 1831.0, + "33": 2101.0, + "34": 1769.0, + "35": 1915.0, + "36": 1885.0, + "37": 2324.0, + "38": 2169.0, + "39": 2300.0, + "40": 2069.0, + "41": 2353.0, + "42": 2236.0, + "43": 1978.0, + "44": 2022.0, + "45": 2103.0, + "46": 2292.0, + "47": 2413.0, + "48": 2305.0, + "49": 2218.0, + "50": 2321.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 522452480.0, + "2": 522452480.0, + "3": 522452480.0, + "4": 522452480.0, + "5": 522452480.0, + "6": 522452480.0, + "7": 522452480.0, + "8": 522452480.0, + "9": 523501056.0, + "10": 522452480.0, + "11": 522452480.0, + "12": 523501056.0, + "13": 522452480.0, + "14": 522452480.0, + "15": 522452480.0, + "16": 522452480.0, + "17": 522452480.0, + "18": 522452480.0, + "19": 523501056.0, + "20": 523501056.0, + "21": 522452480.0, + "22": 522452480.0, + "23": 522452480.0, + "24": 523501056.0, + "25": 522452480.0, + "26": 522452480.0, + "27": 522452480.0, + "28": 522452480.0, + "29": 523501056.0, + "30": 522452480.0, + "31": 522452480.0, + "32": 522452480.0, + "33": 522452480.0, + "34": 522452480.0, + "35": 522452480.0, + "36": 522452480.0, + "37": 522452480.0, + "38": 522452480.0, + "39": 522452480.0, + "40": 522452480.0, + "41": 523371008.0, + "42": 522452480.0, + "43": 522452480.0, + "44": 522452480.0, + "45": 522452480.0, + "46": 523501056.0, + "47": 522452480.0, + "48": 522452480.0, + "49": 523501056.0, + "50": 522452480.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3768846848.0, + "2": 3913263616.0, + "3": 3913263616.0, + "4": 3913263616.0, + "5": 3913263616.0, + "6": 3913263616.0, + "7": 3913263616.0, + "8": 3913263616.0, + "9": 3913263616.0, + "10": 3913263616.0, + "11": 3913263616.0, + "12": 3913263616.0, + "13": 3913263616.0, + "14": 3913263616.0, + "15": 3913263616.0, + "16": 3913263616.0, + "17": 3913263616.0, + "18": 3913263616.0, + "19": 3913263616.0, + "20": 3913263616.0, + "21": 3913263616.0, + "22": 3913263616.0, + "23": 3913263616.0, + "24": 3913263616.0, + "25": 3913263616.0, + "26": 3913263616.0, + "27": 3913263616.0, + "28": 3913263616.0, + "29": 3913263616.0, + "30": 3913263616.0, + "31": 3913263616.0, + "32": 3913263616.0, + "33": 3913263616.0, + "34": 3913263616.0, + "35": 3913263616.0, + "36": 3913263616.0, + "37": 3913263616.0, + "38": 3913263616.0, + "39": 3913263616.0, + "40": 3913263616.0, + "41": 3913263616.0, + "42": 3913263616.0, + "43": 3913263616.0, + "44": 3913263616.0, + "45": 3913263616.0, + "46": 3913263616.0, + "47": 3913263616.0, + "48": 3913263616.0, + "49": 3913263616.0, + "50": 3913263616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.5204, + "2": 0.1877, + "3": 0.15851, + "4": 0.15284, + "5": 0.15092, + "6": 0.15084, + "7": 0.14786, + "8": 0.14787, + "9": 0.14698, + "10": 0.15104, + "11": 0.29695, + "12": 0.14795, + "13": 0.14771, + "14": 0.14662, + "15": 0.14409, + "16": 0.14378, + "17": 0.14431, + "18": 0.14329, + "19": 0.14334, + "20": 0.14441, + "21": 0.28011, + "22": 0.14378, + "23": 0.14643, + "24": 0.14572, + "25": 0.14331, + "26": 0.14307, + "27": 0.14541, + "28": 0.14512, + "29": 0.14536, + "30": 0.14358, + "31": 0.28944, + "32": 0.14533, + "33": 0.14477, + "34": 0.14423, + "35": 0.14395, + "36": 0.14486, + "37": 0.14319, + "38": 0.14455, + "39": 0.14454, + "40": 0.14537, + "41": 0.29312, + "42": 0.14458, + "43": 0.14749, + "44": 0.14448, + "45": 0.14501, + "46": 0.14588, + "47": 0.14249, + "48": 0.14564, + "49": 0.14388, + "50": 0.14222 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json new file mode 100644 index 00000000000..59dee721816 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81271, + "4": 10.78877, + "5": 10.85169, + "6": 10.8704, + "7": 10.831, + "8": 10.83427, + "9": 10.83995, + "10": 10.78684, + "11": 10.88021, + "12": 10.85971, + "13": 10.86589, + "14": 10.87818, + "15": 10.79463, + "16": 10.79607, + "17": 10.7688, + "18": 10.81045, + "19": 10.79836, + "20": 10.69045, + "21": 10.67932, + "22": 10.52101, + "23": 10.70743, + "24": 10.57665, + "25": 10.52275, + "26": 10.595, + "27": 10.5855, + "28": 10.56131, + "29": 10.56894, + "30": 10.34527, + "31": 10.10019, + "32": 10.45229, + "33": 10.44356, + "34": 10.20397, + "35": 10.25844, + "36": 10.2103, + "37": 10.32252, + "38": 10.1661, + "39": 10.38156, + "40": 10.07025, + "41": 10.13542, + "42": 10.19416, + "43": 9.80626, + "44": 9.92627, + "45": 9.8024, + "46": 9.79983, + "47": 10.11662, + "48": 9.81307, + "49": 9.50044, + "50": 9.87631, + "51": 9.82781, + "52": 9.71723, + "53": 10.03979, + "54": 9.92177, + "55": 9.85515, + "56": 9.59253, + "57": 9.44144, + "58": 9.79602, + "59": 9.55567, + "60": 9.4664, + "61": 9.6666, + "62": 9.95363, + "63": 9.33626, + "64": 9.74152, + "65": 8.9178, + "66": 9.66632, + "67": 9.34424, + "68": 9.75273, + "69": 9.75727, + "70": 9.69242, + "71": 9.5868, + "72": 9.55099, + "73": 9.46289, + "74": 8.90671, + "75": 9.37793, + "76": 9.04952, + "77": 10.0301, + "78": 9.69192, + "79": 9.33464, + "80": 9.3667, + "81": 9.44418, + "82": 9.66164, + "83": 9.27209, + "84": 9.38066, + "85": 9.57618, + "86": 9.0424, + "87": 9.55703, + "88": 9.70385, + "89": 9.56619, + "90": 9.77295, + "91": 9.29396, + "92": 9.31912, + "93": 9.03406, + "94": 8.78526, + "95": 9.46938, + "96": 9.47497, + "97": 9.25688, + "98": 9.61835, + "99": 8.83233, + "100": 9.34557 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1668.0, + "4": 1721.0, + "5": 1897.0, + "6": 1823.0, + "7": 1719.0, + "8": 1637.0, + "9": 1742.0, + "10": 1358.0, + "11": 1882.0, + "12": 1781.0, + "13": 1847.0, + "14": 1753.0, + "15": 1883.0, + "16": 1755.0, + "17": 1752.0, + "18": 1683.0, + "19": 1817.0, + "20": 1663.0, + "21": 1795.0, + "22": 1698.0, + "23": 1996.0, + "24": 1620.0, + "25": 1658.0, + "26": 1727.0, + "27": 1781.0, + "28": 2085.0, + "29": 1952.0, + "30": 1821.0, + "31": 1646.0, + "32": 1879.0, + "33": 2034.0, + "34": 1861.0, + "35": 1834.0, + "36": 1913.0, + "37": 2333.0, + "38": 2070.0, + "39": 2245.0, + "40": 2126.0, + "41": 2311.0, + "42": 2213.0, + "43": 1907.0, + "44": 1951.0, + "45": 2001.0, + "46": 2218.0, + "47": 2533.0, + "48": 2436.0, + "49": 2188.0, + "50": 2342.0, + "51": 2562.0, + "52": 2529.0, + "53": 3031.0, + "54": 2744.0, + "55": 2264.0, + "56": 2794.0, + "57": 2183.0, + "58": 2882.0, + "59": 2769.0, + "60": 2399.0, + "61": 3031.0, + "62": 2706.0, + "63": 2388.0, + "64": 3046.0, + "65": 2597.0, + "66": 3092.0, + "67": 2730.0, + "68": 2858.0, + "69": 2982.0, + "70": 3202.0, + "71": 2964.0, + "72": 2450.0, + "73": 2817.0, + "74": 1834.0, + "75": 2609.0, + "76": 3000.0, + "77": 3180.0, + "78": 3113.0, + "79": 3145.0, + "80": 3258.0, + "81": 3645.0, + "82": 3075.0, + "83": 2812.0, + "84": 3295.0, + "85": 3368.0, + "86": 2730.0, + "87": 3717.0, + "88": 3056.0, + "89": 3252.0, + "90": 2954.0, + "91": 2798.0, + "92": 3089.0, + "93": 2742.0, + "94": 3420.0, + "95": 3225.0, + "96": 3362.0, + "97": 3118.0, + "98": 3671.0, + "99": 3341.0, + "100": 3428.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2366437376.0, + "2": 2648148992.0, + "3": 2648148992.0, + "4": 2648148992.0, + "5": 2648148992.0, + "6": 2648148992.0, + "7": 2648148992.0, + "8": 2648148992.0, + "9": 2648148992.0, + "10": 2648148992.0, + "11": 2648148992.0, + "12": 2648148992.0, + "13": 2648148992.0, + "14": 2648148992.0, + "15": 2648148992.0, + "16": 2648148992.0, + "17": 2648148992.0, + "18": 2648148992.0, + "19": 2648148992.0, + "20": 2648148992.0, + "21": 2648148992.0, + "22": 2648148992.0, + "23": 2648148992.0, + "24": 2648148992.0, + "25": 2648148992.0, + "26": 2648148992.0, + "27": 2648148992.0, + "28": 2648148992.0, + "29": 2648148992.0, + "30": 2648148992.0, + "31": 2648148992.0, + "32": 2648148992.0, + "33": 2648148992.0, + "34": 2648148992.0, + "35": 2648148992.0, + "36": 2648148992.0, + "37": 2648148992.0, + "38": 2648148992.0, + "39": 2648148992.0, + "40": 2648148992.0, + "41": 2648148992.0, + "42": 2648148992.0, + "43": 2648148992.0, + "44": 2648148992.0, + "45": 2648148992.0, + "46": 2648148992.0, + "47": 2648148992.0, + "48": 2648148992.0, + "49": 2648148992.0, + "50": 2648148992.0, + "51": 2648148992.0, + "52": 2648148992.0, + "53": 2648148992.0, + "54": 2648148992.0, + "55": 2648148992.0, + "56": 2648148992.0, + "57": 2648148992.0, + "58": 2648148992.0, + "59": 2648148992.0, + "60": 2648148992.0, + "61": 2648148992.0, + "62": 2648148992.0, + "63": 2648148992.0, + "64": 2648148992.0, + "65": 2648148992.0, + "66": 2648148992.0, + "67": 2648148992.0, + "68": 2648148992.0, + "69": 2648148992.0, + "70": 2648148992.0, + "71": 2648148992.0, + "72": 2648148992.0, + "73": 2648148992.0, + "74": 2648148992.0, + "75": 2648148992.0, + "76": 2648148992.0, + "77": 2648148992.0, + "78": 2648148992.0, + "79": 2648148992.0, + "80": 2648148992.0, + "81": 2648148992.0, + "82": 2648148992.0, + "83": 2648148992.0, + "84": 2648148992.0, + "85": 2648148992.0, + "86": 2648148992.0, + "87": 2648148992.0, + "88": 2648148992.0, + "89": 2648148992.0, + "90": 2648148992.0, + "91": 2648148992.0, + "92": 2648148992.0, + "93": 2648148992.0, + "94": 2648148992.0, + "95": 2648148992.0, + "96": 2648148992.0, + "97": 2648148992.0, + "98": 2648148992.0, + "99": 2648148992.0, + "100": 2648148992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.77273, + "2": 0.15704, + "3": 0.19484, + "4": 0.13176, + "5": 0.13019, + "6": 0.12976, + "7": 0.1302, + "8": 0.12925, + "9": 0.12988, + "10": 0.13099, + "11": 0.13015, + "12": 0.1297, + "13": 0.12988, + "14": 0.13024, + "15": 0.12985, + "16": 0.12971, + "17": 0.12961, + "18": 0.1302, + "19": 0.12963, + "20": 0.12994, + "21": 0.1299, + "22": 0.13037, + "23": 0.13043, + "24": 0.12989, + "25": 0.13018, + "26": 0.13019, + "27": 0.12985, + "28": 0.13014, + "29": 0.13068, + "30": 0.13099, + "31": 0.13197, + "32": 0.13151, + "33": 0.13168, + "34": 0.1303, + "35": 0.13073, + "36": 0.13088, + "37": 0.1307, + "38": 0.13091, + "39": 0.13292, + "40": 0.13172, + "41": 0.134, + "42": 0.13157, + "43": 0.13272, + "44": 0.13144, + "45": 0.13142, + "46": 0.133, + "47": 0.13069, + "48": 0.13192, + "49": 0.13124, + "50": 0.13106, + "51": 0.13227, + "52": 0.13218, + "53": 0.13063, + "54": 0.13182, + "55": 0.13138, + "56": 0.13226, + "57": 0.13156, + "58": 0.13127, + "59": 0.13198, + "60": 0.13133, + "61": 0.13107, + "62": 0.13121, + "63": 0.13141, + "64": 0.13149, + "65": 0.13192, + "66": 0.13188, + "67": 0.13167, + "68": 0.1319, + "69": 0.1318, + "70": 0.13204, + "71": 0.13077, + "72": 0.13205, + "73": 0.13251, + "74": 0.13212, + "75": 0.13195, + "76": 0.13016, + "77": 0.12994, + "78": 0.13184, + "79": 0.13131, + "80": 0.13153, + "81": 0.13219, + "82": 0.13088, + "83": 0.13074, + "84": 0.13035, + "85": 0.12939, + "86": 0.12953, + "87": 0.1294, + "88": 0.12951, + "89": 0.13007, + "90": 0.12893, + "91": 0.13881, + "92": 0.1299, + "93": 0.12993, + "94": 0.13022, + "95": 0.1304, + "96": 0.12965, + "97": 0.13013, + "98": 0.1306, + "99": 0.12958, + "100": 0.13011 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json index 855eefe74c4..6081b627567 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85285, "10": 10.78449, "15": 10.79226, "20": 10.69196, "25": 10.52318, "30": 10.34505, "35": 10.25888, "40": 10.07032, "45": 9.803, "50": 9.87675, "55": 9.85523, "60": 9.46634, "65": 8.91657, "70": 9.69276, "75": 9.37813, "80": 9.368, "85": 9.57598, "90": 9.77247, "95": 9.46912, "100": 9.34574}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1915.0, "10": 1361.0, "15": 1831.0, "20": 1695.0, "25": 1650.0, "30": 1861.0, "35": 1916.0, "40": 2109.0, "45": 2098.0, "50": 2424.0, "55": 2309.0, "60": 2432.0, "65": 2660.0, "70": 3132.0, "75": 2555.0, "80": 3370.0, "85": 3434.0, "90": 2868.0, "95": 3273.0, "100": 3359.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0, "55": 732811264.0, "60": 732811264.0, "65": 732811264.0, "70": 732811264.0, "75": 732811264.0, "80": 732811264.0, "85": 732811264.0, "90": 732811264.0, "95": 732811264.0, "100": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0, "55": 4122703872.0, "60": 4122703872.0, "65": 4122703872.0, "70": 4122703872.0, "75": 4122703872.0, "80": 4122703872.0, "85": 4122703872.0, "90": 4122703872.0, "95": 4122703872.0, "100": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 21.12849, "5": 0.1796, "10": 0.17779, "15": 0.17478, "20": 0.17462, "25": 0.17506, "30": 0.17537, "35": 0.17332, "40": 0.17629, "45": 0.17732, "50": 0.17437, "55": 0.1738, "60": 0.17515, "65": 0.17451, "70": 0.17318, "75": 0.1729, "80": 0.17444, "85": 0.17406, "90": 0.17228, "95": 0.17346, "100": 0.17333}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81271, + "4": 10.78877, + "5": 10.85169, + "6": 10.8704, + "7": 10.831, + "8": 10.83427, + "9": 10.83995, + "10": 10.78684, + "11": 10.88021, + "12": 10.85971, + "13": 10.86589, + "14": 10.87818, + "15": 10.79463, + "16": 10.79607, + "17": 10.7688, + "18": 10.81045, + "19": 10.79836, + "20": 10.69045, + "21": 10.67932, + "22": 10.52101, + "23": 10.70743, + "24": 10.57665, + "25": 10.52275, + "26": 10.595, + "27": 10.5855, + "28": 10.56131, + "29": 10.56894, + "30": 10.34527, + "31": 10.10019, + "32": 10.45229, + "33": 10.44356, + "34": 10.20397, + "35": 10.25844, + "36": 10.2103, + "37": 10.32252, + "38": 10.1661, + "39": 10.38156, + "40": 10.07025, + "41": 10.13542, + "42": 10.19416, + "43": 9.80626, + "44": 9.92627, + "45": 9.8024, + "46": 9.79983, + "47": 10.11662, + "48": 9.81307, + "49": 9.50044, + "50": 9.87631, + "51": 9.82781, + "52": 9.71723, + "53": 10.03979, + "54": 9.92177, + "55": 9.85515, + "56": 9.59253, + "57": 9.44144, + "58": 9.79602, + "59": 9.55567, + "60": 9.4664, + "61": 9.6666, + "62": 9.95363, + "63": 9.33626, + "64": 9.74152, + "65": 8.9178, + "66": 9.66632, + "67": 9.34424, + "68": 9.75273, + "69": 9.75727, + "70": 9.69242, + "71": 9.5868, + "72": 9.55099, + "73": 9.46289, + "74": 8.90671, + "75": 9.37793, + "76": 9.04952, + "77": 10.0301, + "78": 9.69192, + "79": 9.33464, + "80": 9.3667, + "81": 9.44418, + "82": 9.66164, + "83": 9.27209, + "84": 9.38066, + "85": 9.57618, + "86": 9.0424, + "87": 9.55703, + "88": 9.70385, + "89": 9.56619, + "90": 9.77295, + "91": 9.29396, + "92": 9.31912, + "93": 9.03406, + "94": 8.78526, + "95": 9.46938, + "96": 9.47497, + "97": 9.25688, + "98": 9.61835, + "99": 8.83233, + "100": 9.34557 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1668.0, + "4": 1721.0, + "5": 1897.0, + "6": 1823.0, + "7": 1719.0, + "8": 1637.0, + "9": 1742.0, + "10": 1358.0, + "11": 1882.0, + "12": 1781.0, + "13": 1847.0, + "14": 1753.0, + "15": 1883.0, + "16": 1755.0, + "17": 1752.0, + "18": 1683.0, + "19": 1817.0, + "20": 1663.0, + "21": 1795.0, + "22": 1698.0, + "23": 1996.0, + "24": 1620.0, + "25": 1658.0, + "26": 1727.0, + "27": 1781.0, + "28": 2085.0, + "29": 1952.0, + "30": 1821.0, + "31": 1646.0, + "32": 1879.0, + "33": 2034.0, + "34": 1861.0, + "35": 1834.0, + "36": 1913.0, + "37": 2333.0, + "38": 2070.0, + "39": 2245.0, + "40": 2126.0, + "41": 2311.0, + "42": 2213.0, + "43": 1907.0, + "44": 1951.0, + "45": 2001.0, + "46": 2218.0, + "47": 2533.0, + "48": 2436.0, + "49": 2188.0, + "50": 2342.0, + "51": 2562.0, + "52": 2529.0, + "53": 3031.0, + "54": 2744.0, + "55": 2264.0, + "56": 2794.0, + "57": 2183.0, + "58": 2882.0, + "59": 2769.0, + "60": 2399.0, + "61": 3031.0, + "62": 2706.0, + "63": 2388.0, + "64": 3046.0, + "65": 2597.0, + "66": 3092.0, + "67": 2730.0, + "68": 2858.0, + "69": 2982.0, + "70": 3202.0, + "71": 2964.0, + "72": 2450.0, + "73": 2817.0, + "74": 1834.0, + "75": 2609.0, + "76": 3000.0, + "77": 3180.0, + "78": 3113.0, + "79": 3145.0, + "80": 3258.0, + "81": 3645.0, + "82": 3075.0, + "83": 2812.0, + "84": 3295.0, + "85": 3368.0, + "86": 2730.0, + "87": 3717.0, + "88": 3056.0, + "89": 3252.0, + "90": 2954.0, + "91": 2798.0, + "92": 3089.0, + "93": 2742.0, + "94": 3420.0, + "95": 3225.0, + "96": 3362.0, + "97": 3118.0, + "98": 3671.0, + "99": 3341.0, + "100": 3428.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2366437376.0, + "2": 2648148992.0, + "3": 2648148992.0, + "4": 2648148992.0, + "5": 2648148992.0, + "6": 2648148992.0, + "7": 2648148992.0, + "8": 2648148992.0, + "9": 2648148992.0, + "10": 2648148992.0, + "11": 2648148992.0, + "12": 2648148992.0, + "13": 2648148992.0, + "14": 2648148992.0, + "15": 2648148992.0, + "16": 2648148992.0, + "17": 2648148992.0, + "18": 2648148992.0, + "19": 2648148992.0, + "20": 2648148992.0, + "21": 2648148992.0, + "22": 2648148992.0, + "23": 2648148992.0, + "24": 2648148992.0, + "25": 2648148992.0, + "26": 2648148992.0, + "27": 2648148992.0, + "28": 2648148992.0, + "29": 2648148992.0, + "30": 2648148992.0, + "31": 2648148992.0, + "32": 2648148992.0, + "33": 2648148992.0, + "34": 2648148992.0, + "35": 2648148992.0, + "36": 2648148992.0, + "37": 2648148992.0, + "38": 2648148992.0, + "39": 2648148992.0, + "40": 2648148992.0, + "41": 2648148992.0, + "42": 2648148992.0, + "43": 2648148992.0, + "44": 2648148992.0, + "45": 2648148992.0, + "46": 2648148992.0, + "47": 2648148992.0, + "48": 2648148992.0, + "49": 2648148992.0, + "50": 2648148992.0, + "51": 2648148992.0, + "52": 2648148992.0, + "53": 2648148992.0, + "54": 2648148992.0, + "55": 2648148992.0, + "56": 2648148992.0, + "57": 2648148992.0, + "58": 2648148992.0, + "59": 2648148992.0, + "60": 2648148992.0, + "61": 2648148992.0, + "62": 2648148992.0, + "63": 2648148992.0, + "64": 2648148992.0, + "65": 2648148992.0, + "66": 2648148992.0, + "67": 2648148992.0, + "68": 2648148992.0, + "69": 2648148992.0, + "70": 2648148992.0, + "71": 2648148992.0, + "72": 2648148992.0, + "73": 2648148992.0, + "74": 2648148992.0, + "75": 2648148992.0, + "76": 2648148992.0, + "77": 2648148992.0, + "78": 2648148992.0, + "79": 2648148992.0, + "80": 2648148992.0, + "81": 2648148992.0, + "82": 2648148992.0, + "83": 2648148992.0, + "84": 2648148992.0, + "85": 2648148992.0, + "86": 2648148992.0, + "87": 2648148992.0, + "88": 2648148992.0, + "89": 2648148992.0, + "90": 2648148992.0, + "91": 2648148992.0, + "92": 2648148992.0, + "93": 2648148992.0, + "94": 2648148992.0, + "95": 2648148992.0, + "96": 2648148992.0, + "97": 2648148992.0, + "98": 2648148992.0, + "99": 2648148992.0, + "100": 2648148992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.81482, + "2": 0.16445, + "3": 0.16681, + "4": 0.12923, + "5": 0.12855, + "6": 0.12774, + "7": 0.12794, + "8": 0.12857, + "9": 0.12785, + "10": 0.12889, + "11": 0.13344, + "12": 0.1302, + "13": 0.13007, + "14": 0.12962, + "15": 0.13044, + "16": 0.12918, + "17": 0.13075, + "18": 0.13004, + "19": 0.13052, + "20": 0.13025, + "21": 0.12825, + "22": 0.13322, + "23": 0.13274, + "24": 0.13114, + "25": 0.13075, + "26": 0.12979, + "27": 0.13026, + "28": 0.13147, + "29": 0.13072, + "30": 0.13098, + "31": 0.13095, + "32": 0.13054, + "33": 0.13038, + "34": 0.13142, + "35": 0.13065, + "36": 0.12923, + "37": 0.13039, + "38": 0.12981, + "39": 0.12995, + "40": 0.13035, + "41": 0.12966, + "42": 0.13013, + "43": 0.13031, + "44": 0.13066, + "45": 0.12952, + "46": 0.13059, + "47": 0.12932, + "48": 0.13133, + "49": 0.13099, + "50": 0.13032, + "51": 0.13345, + "52": 0.13027, + "53": 0.13035, + "54": 0.13064, + "55": 0.13026, + "56": 0.13053, + "57": 0.13106, + "58": 0.13032, + "59": 0.13178, + "60": 0.13233, + "61": 0.13005, + "62": 0.13045, + "63": 0.13061, + "64": 0.13066, + "65": 0.13102, + "66": 0.13143, + "67": 0.13033, + "68": 0.13066, + "69": 0.12904, + "70": 0.13059, + "71": 0.13052, + "72": 0.13076, + "73": 0.13215, + "74": 0.13173, + "75": 0.13126, + "76": 0.12946, + "77": 0.13071, + "78": 0.12973, + "79": 0.12962, + "80": 0.12976, + "81": 0.12993, + "82": 0.12829, + "83": 0.13132, + "84": 0.1304, + "85": 0.13095, + "86": 0.13112, + "87": 0.12994, + "88": 0.13287, + "89": 0.1284, + "90": 0.1303, + "91": 0.12966, + "92": 0.13139, + "93": 0.12932, + "94": 0.12687, + "95": 0.13012, + "96": 0.12919, + "97": 0.13166, + "98": 0.12958, + "99": 0.13126, + "100": 0.1303 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json new file mode 100644 index 00000000000..f108f63200b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79613, + "17": 10.76887, + "18": 10.81076, + "19": 10.7986, + "20": 10.69061, + "21": 10.67945, + "22": 10.52114, + "23": 10.70755, + "24": 10.57741, + "25": 10.52373, + "26": 10.59597, + "27": 10.58619, + "28": 10.56176, + "29": 10.56945, + "30": 10.34573, + "31": 10.10078, + "32": 10.45392, + "33": 10.4461, + "34": 10.20721, + "35": 10.262, + "36": 10.21312, + "37": 10.32468, + "38": 10.16831, + "39": 10.38374, + "40": 10.07231, + "41": 10.13763, + "42": 10.19765, + "43": 9.81155, + "44": 9.93311, + "45": 9.8104, + "46": 9.80854, + "47": 10.12558, + "48": 9.82105, + "49": 9.50764, + "50": 9.88382, + "51": 9.83549, + "52": 9.72516, + "53": 10.04799, + "54": 9.93011, + "55": 9.8636, + "56": 9.60217, + "57": 9.45187, + "58": 9.8078, + "59": 9.56783, + "60": 9.47966, + "61": 9.67984, + "62": 9.96754, + "63": 9.35113, + "64": 9.75623, + "65": 8.9318, + "66": 9.68107, + "67": 9.35956, + "68": 9.76948, + "69": 9.77492, + "70": 9.71182, + "71": 9.60632, + "72": 9.57129, + "73": 9.48392, + "74": 8.92911, + "75": 9.40028, + "76": 9.07194, + "77": 10.05252, + "78": 9.71494, + "79": 9.35747, + "80": 9.38946, + "81": 9.46791, + "82": 9.68508, + "83": 9.29588, + "84": 9.40522, + "85": 9.60163, + "86": 9.06713, + "87": 9.58402, + "88": 9.73304, + "89": 9.59526, + "90": 9.80555, + "91": 9.32604, + "92": 9.35323, + "93": 9.06915, + "94": 8.82268, + "95": 9.50858, + "96": 9.51584, + "97": 9.2976, + "98": 9.66184, + "99": 8.87662, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1728.0, + "16": 1773.0, + "17": 1805.0, + "18": 1702.0, + "19": 1803.0, + "20": 1627.0, + "21": 1822.0, + "22": 1748.0, + "23": 1938.0, + "24": 1642.0, + "25": 1696.0, + "26": 1760.0, + "27": 1809.0, + "28": 2025.0, + "29": 1900.0, + "30": 1902.0, + "31": 1645.0, + "32": 1876.0, + "33": 2105.0, + "34": 1881.0, + "35": 1913.0, + "36": 1864.0, + "37": 2322.0, + "38": 2194.0, + "39": 2318.0, + "40": 2010.0, + "41": 2358.0, + "42": 2155.0, + "43": 1980.0, + "44": 2105.0, + "45": 2082.0, + "46": 2221.0, + "47": 2537.0, + "48": 2367.0, + "49": 2190.0, + "50": 2352.0, + "51": 2441.0, + "52": 2482.0, + "53": 2916.0, + "54": 2550.0, + "55": 2347.0, + "56": 2765.0, + "57": 2116.0, + "58": 2968.0, + "59": 2810.0, + "60": 2384.0, + "61": 2912.0, + "62": 2554.0, + "63": 2364.0, + "64": 3035.0, + "65": 2648.0, + "66": 2979.0, + "67": 2741.0, + "68": 2799.0, + "69": 3071.0, + "70": 3098.0, + "71": 2950.0, + "72": 2342.0, + "73": 2829.0, + "74": 1840.0, + "75": 2426.0, + "76": 2941.0, + "77": 3245.0, + "78": 3272.0, + "79": 3066.0, + "80": 3221.0, + "81": 3565.0, + "82": 3162.0, + "83": 2876.0, + "84": 3180.0, + "85": 3410.0, + "86": 2778.0, + "87": 3752.0, + "88": 2995.0, + "89": 3264.0, + "90": 2940.0, + "91": 2791.0, + "92": 3118.0, + "93": 2634.0, + "94": 3464.0, + "95": 3344.0, + "96": 3499.0, + "97": 3122.0, + "98": 3568.0, + "99": 3272.0, + "100": 3476.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 733860352.0, + "2": 733860352.0, + "3": 733860352.0, + "4": 733860352.0, + "5": 733860352.0, + "6": 733860352.0, + "7": 733860352.0, + "8": 733860352.0, + "9": 733860352.0, + "10": 733860352.0, + "11": 733860352.0, + "12": 733860352.0, + "13": 733860352.0, + "14": 733860352.0, + "15": 733860352.0, + "16": 733860352.0, + "17": 733860352.0, + "18": 733860352.0, + "19": 733860352.0, + "20": 733860352.0, + "21": 733860352.0, + "22": 733860352.0, + "23": 733860352.0, + "24": 733860352.0, + "25": 733860352.0, + "26": 733860352.0, + "27": 733860352.0, + "28": 733860352.0, + "29": 733860352.0, + "30": 733860352.0, + "31": 733860352.0, + "32": 733860352.0, + "33": 733860352.0, + "34": 733860352.0, + "35": 733860352.0, + "36": 733860352.0, + "37": 733860352.0, + "38": 733860352.0, + "39": 733860352.0, + "40": 733860352.0, + "41": 733860352.0, + "42": 733860352.0, + "43": 733860352.0, + "44": 733860352.0, + "45": 733860352.0, + "46": 733860352.0, + "47": 733860352.0, + "48": 733860352.0, + "49": 733860352.0, + "50": 733860352.0, + "51": 733860352.0, + "52": 733860352.0, + "53": 733860352.0, + "54": 733860352.0, + "55": 733860352.0, + "56": 733860352.0, + "57": 733860352.0, + "58": 733860352.0, + "59": 733860352.0, + "60": 733860352.0, + "61": 733860352.0, + "62": 733860352.0, + "63": 733860352.0, + "64": 733860352.0, + "65": 733860352.0, + "66": 733860352.0, + "67": 733860352.0, + "68": 733860352.0, + "69": 733860352.0, + "70": 733860352.0, + "71": 733860352.0, + "72": 733860352.0, + "73": 733860352.0, + "74": 733860352.0, + "75": 733860352.0, + "76": 733860352.0, + "77": 733860352.0, + "78": 733860352.0, + "79": 733860352.0, + "80": 733860352.0, + "81": 733860352.0, + "82": 733860352.0, + "83": 733860352.0, + "84": 733860352.0, + "85": 733860352.0, + "86": 733860352.0, + "87": 733860352.0, + "88": 733860352.0, + "89": 733860352.0, + "90": 733860352.0, + "91": 733860352.0, + "92": 733860352.0, + "93": 733860352.0, + "94": 733860352.0, + "95": 733860352.0, + "96": 733860352.0, + "97": 733860352.0, + "98": 733860352.0, + "99": 733860352.0, + "100": 733860352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2367879168.0, + "2": 2651687936.0, + "3": 2651687936.0, + "4": 2651687936.0, + "5": 2651687936.0, + "6": 2651687936.0, + "7": 2651687936.0, + "8": 2651687936.0, + "9": 2651687936.0, + "10": 2651687936.0, + "11": 2651687936.0, + "12": 2651687936.0, + "13": 2651687936.0, + "14": 2651687936.0, + "15": 2651687936.0, + "16": 2651687936.0, + "17": 2651687936.0, + "18": 2651687936.0, + "19": 2651687936.0, + "20": 2651687936.0, + "21": 2651687936.0, + "22": 2651687936.0, + "23": 2651687936.0, + "24": 2651687936.0, + "25": 2651687936.0, + "26": 2651687936.0, + "27": 2651687936.0, + "28": 2651687936.0, + "29": 2651687936.0, + "30": 2651687936.0, + "31": 2651687936.0, + "32": 2651687936.0, + "33": 2651687936.0, + "34": 2651687936.0, + "35": 2651687936.0, + "36": 2651687936.0, + "37": 2651687936.0, + "38": 2651687936.0, + "39": 2651687936.0, + "40": 2651687936.0, + "41": 2651687936.0, + "42": 2651687936.0, + "43": 2651687936.0, + "44": 2651687936.0, + "45": 2651687936.0, + "46": 2651687936.0, + "47": 2651687936.0, + "48": 2651687936.0, + "49": 2651687936.0, + "50": 2651687936.0, + "51": 2651687936.0, + "52": 2651687936.0, + "53": 2651687936.0, + "54": 2651687936.0, + "55": 2651687936.0, + "56": 2651687936.0, + "57": 2651687936.0, + "58": 2651687936.0, + "59": 2651687936.0, + "60": 2651687936.0, + "61": 2651687936.0, + "62": 2651687936.0, + "63": 2651687936.0, + "64": 2651687936.0, + "65": 2651687936.0, + "66": 2651687936.0, + "67": 2651687936.0, + "68": 2651687936.0, + "69": 2651687936.0, + "70": 2651687936.0, + "71": 2651687936.0, + "72": 2651687936.0, + "73": 2651687936.0, + "74": 2651687936.0, + "75": 2651687936.0, + "76": 2651687936.0, + "77": 2651687936.0, + "78": 2651687936.0, + "79": 2651687936.0, + "80": 2651687936.0, + "81": 2651687936.0, + "82": 2651687936.0, + "83": 2651687936.0, + "84": 2651687936.0, + "85": 2651687936.0, + "86": 2651687936.0, + "87": 2651687936.0, + "88": 2651687936.0, + "89": 2651687936.0, + "90": 2651687936.0, + "91": 2651687936.0, + "92": 2651687936.0, + "93": 2651687936.0, + "94": 2651687936.0, + "95": 2651687936.0, + "96": 2651687936.0, + "97": 2651687936.0, + "98": 2651687936.0, + "99": 2651687936.0, + "100": 2651687936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.1008, + "2": 0.16051, + "3": 0.12978, + "4": 0.12855, + "5": 0.12836, + "6": 0.12718, + "7": 0.12817, + "8": 0.12827, + "9": 0.12773, + "10": 0.12934, + "11": 0.1284, + "12": 0.1278, + "13": 0.12824, + "14": 0.12897, + "15": 0.12788, + "16": 0.12662, + "17": 0.12751, + "18": 0.12678, + "19": 0.12784, + "20": 0.12756, + "21": 0.12782, + "22": 0.12765, + "23": 0.12695, + "24": 0.12621, + "25": 0.12639, + "26": 0.12652, + "27": 0.1261, + "28": 0.12599, + "29": 0.12679, + "30": 0.12648, + "31": 0.12791, + "32": 0.1267, + "33": 0.12736, + "34": 0.1275, + "35": 0.12674, + "36": 0.12623, + "37": 0.12561, + "38": 0.12629, + "39": 0.12735, + "40": 0.12739, + "41": 0.12784, + "42": 0.12763, + "43": 0.12841, + "44": 0.12666, + "45": 0.12797, + "46": 0.12722, + "47": 0.12583, + "48": 0.1271, + "49": 0.12675, + "50": 0.12769, + "51": 0.13852, + "52": 0.1338, + "53": 0.1334, + "54": 0.13466, + "55": 0.13471, + "56": 0.13266, + "57": 0.13116, + "58": 0.13405, + "59": 0.13356, + "60": 0.13304, + "61": 0.13328, + "62": 0.13243, + "63": 0.13255, + "64": 0.13344, + "65": 0.13433, + "66": 0.13486, + "67": 0.13338, + "68": 0.13313, + "69": 0.13327, + "70": 0.1324, + "71": 0.13325, + "72": 0.13418, + "73": 0.1341, + "74": 0.1334, + "75": 0.13238, + "76": 0.13198, + "77": 0.13412, + "78": 0.1335, + "79": 0.13208, + "80": 0.13334, + "81": 0.13338, + "82": 0.13187, + "83": 0.13324, + "84": 0.13268, + "85": 0.13362, + "86": 0.13282, + "87": 0.13325, + "88": 0.13348, + "89": 0.13361, + "90": 0.13267, + "91": 0.13322, + "92": 0.13404, + "93": 0.13424, + "94": 0.13249, + "95": 0.1323, + "96": 0.13217, + "97": 0.16026, + "98": 0.13491, + "99": 0.13704, + "100": 0.13716 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json index fbeaf7f4f0d..0cb12854799 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69211, "25": 10.52412, "30": 10.34552, "35": 10.26239, "40": 10.07241, "45": 9.81101, "50": 9.88422, "55": 9.86374, "60": 9.47965, "65": 8.93063, "70": 9.71215, "75": 9.40048, "80": 9.39077, "85": 9.60141, "90": 9.80501, "95": 9.50816, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1601.0, "25": 1635.0, "30": 1936.0, "35": 1908.0, "40": 2100.0, "45": 2098.0, "50": 2333.0, "55": 2260.0, "60": 2399.0, "65": 2656.0, "70": 3077.0, "75": 2547.0, "80": 3315.0, "85": 3371.0, "90": 2943.0, "95": 3457.0, "100": 3292.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 732811264.0, "5": 732811264.0, "10": 732811264.0, "15": 732811264.0, "20": 732811264.0, "25": 732811264.0, "30": 732811264.0, "35": 732811264.0, "40": 732811264.0, "45": 732811264.0, "50": 732811264.0, "55": 732811264.0, "60": 732811264.0, "65": 732811264.0, "70": 732811264.0, "75": 732811264.0, "80": 732811264.0, "85": 732811264.0, "90": 732811264.0, "95": 732811264.0, "100": 732811264.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3838895104.0, "5": 4122703872.0, "10": 4122703872.0, "15": 4122703872.0, "20": 4122703872.0, "25": 4122703872.0, "30": 4122703872.0, "35": 4122703872.0, "40": 4122703872.0, "45": 4122703872.0, "50": 4122703872.0, "55": 4122703872.0, "60": 4122703872.0, "65": 4122703872.0, "70": 4122703872.0, "75": 4122703872.0, "80": 4122703872.0, "85": 4122703872.0, "90": 4122703872.0, "95": 4122703872.0, "100": 4122703872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 24.14685, "5": 0.1855, "10": 0.18669, "15": 0.18585, "20": 0.18594, "25": 0.18564, "30": 0.18382, "35": 0.18679, "40": 0.1866, "45": 0.18389, "50": 0.18294, "55": 0.17556, "60": 0.17509, "65": 0.17542, "70": 0.17558, "75": 0.176, "80": 0.17523, "85": 0.17384, "90": 0.17335, "95": 0.17222, "100": 0.17356}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79613, + "17": 10.76887, + "18": 10.81076, + "19": 10.7986, + "20": 10.69061, + "21": 10.67945, + "22": 10.52114, + "23": 10.70755, + "24": 10.57741, + "25": 10.52373, + "26": 10.59597, + "27": 10.58619, + "28": 10.56176, + "29": 10.56945, + "30": 10.34573, + "31": 10.10078, + "32": 10.45392, + "33": 10.4461, + "34": 10.20721, + "35": 10.262, + "36": 10.21312, + "37": 10.32468, + "38": 10.16831, + "39": 10.38374, + "40": 10.07231, + "41": 10.13763, + "42": 10.19765, + "43": 9.81155, + "44": 9.93311, + "45": 9.8104, + "46": 9.80854, + "47": 10.12558, + "48": 9.82105, + "49": 9.50764, + "50": 9.88382, + "51": 9.83549, + "52": 9.72516, + "53": 10.04799, + "54": 9.93011, + "55": 9.8636, + "56": 9.60217, + "57": 9.45187, + "58": 9.8078, + "59": 9.56783, + "60": 9.47966, + "61": 9.67984, + "62": 9.96754, + "63": 9.35113, + "64": 9.75623, + "65": 8.9318, + "66": 9.68107, + "67": 9.35956, + "68": 9.76948, + "69": 9.77492, + "70": 9.71182, + "71": 9.60632, + "72": 9.57129, + "73": 9.48392, + "74": 8.92911, + "75": 9.40028, + "76": 9.07194, + "77": 10.05252, + "78": 9.71494, + "79": 9.35747, + "80": 9.38946, + "81": 9.46791, + "82": 9.68508, + "83": 9.29588, + "84": 9.40522, + "85": 9.60163, + "86": 9.06713, + "87": 9.58402, + "88": 9.73304, + "89": 9.59526, + "90": 9.80555, + "91": 9.32604, + "92": 9.35323, + "93": 9.06915, + "94": 8.82268, + "95": 9.50858, + "96": 9.51584, + "97": 9.2976, + "98": 9.66184, + "99": 8.87662, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1728.0, + "16": 1773.0, + "17": 1805.0, + "18": 1702.0, + "19": 1803.0, + "20": 1627.0, + "21": 1822.0, + "22": 1748.0, + "23": 1938.0, + "24": 1642.0, + "25": 1696.0, + "26": 1760.0, + "27": 1809.0, + "28": 2025.0, + "29": 1900.0, + "30": 1902.0, + "31": 1645.0, + "32": 1876.0, + "33": 2105.0, + "34": 1881.0, + "35": 1913.0, + "36": 1864.0, + "37": 2322.0, + "38": 2194.0, + "39": 2318.0, + "40": 2010.0, + "41": 2358.0, + "42": 2155.0, + "43": 1980.0, + "44": 2105.0, + "45": 2082.0, + "46": 2221.0, + "47": 2537.0, + "48": 2367.0, + "49": 2190.0, + "50": 2352.0, + "51": 2441.0, + "52": 2482.0, + "53": 2916.0, + "54": 2550.0, + "55": 2347.0, + "56": 2765.0, + "57": 2116.0, + "58": 2968.0, + "59": 2810.0, + "60": 2384.0, + "61": 2912.0, + "62": 2554.0, + "63": 2364.0, + "64": 3035.0, + "65": 2648.0, + "66": 2979.0, + "67": 2741.0, + "68": 2799.0, + "69": 3071.0, + "70": 3098.0, + "71": 2950.0, + "72": 2342.0, + "73": 2829.0, + "74": 1840.0, + "75": 2426.0, + "76": 2941.0, + "77": 3245.0, + "78": 3272.0, + "79": 3066.0, + "80": 3221.0, + "81": 3565.0, + "82": 3162.0, + "83": 2876.0, + "84": 3180.0, + "85": 3410.0, + "86": 2778.0, + "87": 3752.0, + "88": 2995.0, + "89": 3264.0, + "90": 2940.0, + "91": 2791.0, + "92": 3118.0, + "93": 2634.0, + "94": 3464.0, + "95": 3344.0, + "96": 3499.0, + "97": 3122.0, + "98": 3568.0, + "99": 3272.0, + "100": 3476.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 733860352.0, + "2": 733860352.0, + "3": 733860352.0, + "4": 733860352.0, + "5": 733860352.0, + "6": 733860352.0, + "7": 733860352.0, + "8": 733860352.0, + "9": 733860352.0, + "10": 733860352.0, + "11": 733860352.0, + "12": 733860352.0, + "13": 733860352.0, + "14": 733860352.0, + "15": 733860352.0, + "16": 733860352.0, + "17": 733860352.0, + "18": 733860352.0, + "19": 733860352.0, + "20": 733860352.0, + "21": 733860352.0, + "22": 733860352.0, + "23": 733860352.0, + "24": 733860352.0, + "25": 733860352.0, + "26": 733860352.0, + "27": 733860352.0, + "28": 733860352.0, + "29": 733860352.0, + "30": 733860352.0, + "31": 733860352.0, + "32": 733860352.0, + "33": 733860352.0, + "34": 733860352.0, + "35": 733860352.0, + "36": 733860352.0, + "37": 733860352.0, + "38": 733860352.0, + "39": 733860352.0, + "40": 733860352.0, + "41": 733860352.0, + "42": 733860352.0, + "43": 733860352.0, + "44": 733860352.0, + "45": 733860352.0, + "46": 733860352.0, + "47": 733860352.0, + "48": 733860352.0, + "49": 733860352.0, + "50": 733860352.0, + "51": 733860352.0, + "52": 733860352.0, + "53": 733860352.0, + "54": 733860352.0, + "55": 733860352.0, + "56": 733860352.0, + "57": 733860352.0, + "58": 733860352.0, + "59": 733860352.0, + "60": 733860352.0, + "61": 733860352.0, + "62": 733860352.0, + "63": 733860352.0, + "64": 733860352.0, + "65": 733860352.0, + "66": 733860352.0, + "67": 733860352.0, + "68": 733860352.0, + "69": 733860352.0, + "70": 733860352.0, + "71": 733860352.0, + "72": 733860352.0, + "73": 733860352.0, + "74": 733860352.0, + "75": 733860352.0, + "76": 733860352.0, + "77": 733860352.0, + "78": 733860352.0, + "79": 733860352.0, + "80": 733860352.0, + "81": 733860352.0, + "82": 733860352.0, + "83": 733860352.0, + "84": 733860352.0, + "85": 733860352.0, + "86": 733860352.0, + "87": 733860352.0, + "88": 733860352.0, + "89": 733860352.0, + "90": 733860352.0, + "91": 733860352.0, + "92": 733860352.0, + "93": 733860352.0, + "94": 733860352.0, + "95": 733860352.0, + "96": 733860352.0, + "97": 733860352.0, + "98": 733860352.0, + "99": 733860352.0, + "100": 733860352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2367879168.0, + "2": 2651687936.0, + "3": 2651687936.0, + "4": 2651687936.0, + "5": 2651687936.0, + "6": 2651687936.0, + "7": 2651687936.0, + "8": 2651687936.0, + "9": 2651687936.0, + "10": 2651687936.0, + "11": 2651687936.0, + "12": 2651687936.0, + "13": 2651687936.0, + "14": 2651687936.0, + "15": 2651687936.0, + "16": 2651687936.0, + "17": 2651687936.0, + "18": 2651687936.0, + "19": 2651687936.0, + "20": 2651687936.0, + "21": 2651687936.0, + "22": 2651687936.0, + "23": 2651687936.0, + "24": 2651687936.0, + "25": 2651687936.0, + "26": 2651687936.0, + "27": 2651687936.0, + "28": 2651687936.0, + "29": 2651687936.0, + "30": 2651687936.0, + "31": 2651687936.0, + "32": 2651687936.0, + "33": 2651687936.0, + "34": 2651687936.0, + "35": 2651687936.0, + "36": 2651687936.0, + "37": 2651687936.0, + "38": 2651687936.0, + "39": 2651687936.0, + "40": 2651687936.0, + "41": 2651687936.0, + "42": 2651687936.0, + "43": 2651687936.0, + "44": 2651687936.0, + "45": 2651687936.0, + "46": 2651687936.0, + "47": 2651687936.0, + "48": 2651687936.0, + "49": 2651687936.0, + "50": 2651687936.0, + "51": 2651687936.0, + "52": 2651687936.0, + "53": 2651687936.0, + "54": 2651687936.0, + "55": 2651687936.0, + "56": 2651687936.0, + "57": 2651687936.0, + "58": 2651687936.0, + "59": 2651687936.0, + "60": 2651687936.0, + "61": 2651687936.0, + "62": 2651687936.0, + "63": 2651687936.0, + "64": 2651687936.0, + "65": 2651687936.0, + "66": 2651687936.0, + "67": 2651687936.0, + "68": 2651687936.0, + "69": 2651687936.0, + "70": 2651687936.0, + "71": 2651687936.0, + "72": 2651687936.0, + "73": 2651687936.0, + "74": 2651687936.0, + "75": 2651687936.0, + "76": 2651687936.0, + "77": 2651687936.0, + "78": 2651687936.0, + "79": 2651687936.0, + "80": 2651687936.0, + "81": 2651687936.0, + "82": 2651687936.0, + "83": 2651687936.0, + "84": 2651687936.0, + "85": 2651687936.0, + "86": 2651687936.0, + "87": 2651687936.0, + "88": 2651687936.0, + "89": 2651687936.0, + "90": 2651687936.0, + "91": 2651687936.0, + "92": 2651687936.0, + "93": 2651687936.0, + "94": 2651687936.0, + "95": 2651687936.0, + "96": 2651687936.0, + "97": 2651687936.0, + "98": 2651687936.0, + "99": 2651687936.0, + "100": 2651687936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.6402, + "2": 0.15932, + "3": 0.13183, + "4": 0.12969, + "5": 0.12913, + "6": 0.12942, + "7": 0.12823, + "8": 0.13014, + "9": 0.1305, + "10": 0.13004, + "11": 0.12983, + "12": 0.12943, + "13": 0.12925, + "14": 0.13022, + "15": 0.12947, + "16": 0.12988, + "17": 0.12984, + "18": 0.12989, + "19": 0.12987, + "20": 0.12935, + "21": 0.12974, + "22": 0.12965, + "23": 0.12983, + "24": 0.13037, + "25": 0.1293, + "26": 0.12914, + "27": 0.12908, + "28": 0.12909, + "29": 0.13186, + "30": 0.13433, + "31": 0.13401, + "32": 0.12902, + "33": 0.12808, + "34": 0.12907, + "35": 0.12884, + "36": 0.12913, + "37": 0.12932, + "38": 0.12992, + "39": 0.13072, + "40": 0.13131, + "41": 0.13172, + "42": 0.13072, + "43": 0.13259, + "44": 0.13124, + "45": 0.13129, + "46": 0.1291, + "47": 0.1308, + "48": 0.1301, + "49": 0.12906, + "50": 0.12828, + "51": 0.14265, + "52": 0.12979, + "53": 0.126, + "54": 0.12545, + "55": 0.12582, + "56": 0.12573, + "57": 0.12516, + "58": 0.1252, + "59": 0.12598, + "60": 0.12562, + "61": 0.12544, + "62": 0.12472, + "63": 0.12548, + "64": 0.12537, + "65": 0.12534, + "66": 0.12474, + "67": 0.12528, + "68": 0.12481, + "69": 0.12531, + "70": 0.12547, + "71": 0.12492, + "72": 0.12533, + "73": 0.12583, + "74": 0.1253, + "75": 0.12453, + "76": 0.12486, + "77": 0.12501, + "78": 0.12491, + "79": 0.12247, + "80": 0.1223, + "81": 0.1243, + "82": 0.12257, + "83": 0.12179, + "84": 0.12254, + "85": 0.12231, + "86": 0.12263, + "87": 0.12152, + "88": 0.12188, + "89": 0.1228, + "90": 0.12133, + "91": 0.1216, + "92": 0.12133, + "93": 0.12135, + "94": 0.12216, + "95": 0.12141, + "96": 0.12205, + "97": 0.12356, + "98": 0.12174, + "99": 0.12252, + "100": 0.1222 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..9798744d286 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 520868352.0, + "2": 520868352.0, + "3": 520868352.0, + "4": 520868352.0, + "5": 520868352.0, + "6": 520868352.0, + "7": 520868352.0, + "8": 520868352.0, + "9": 520868352.0, + "10": 520868352.0, + "11": 520868352.0, + "12": 520868352.0, + "13": 520868352.0, + "14": 520868352.0, + "15": 520868352.0, + "16": 520868352.0, + "17": 520868352.0, + "18": 520868352.0, + "19": 520868352.0, + "20": 520868352.0, + "21": 520868352.0, + "22": 520868352.0, + "23": 520868352.0, + "24": 520868352.0, + "25": 520868352.0, + "26": 520868352.0, + "27": 520868352.0, + "28": 520868352.0, + "29": 520868352.0, + "30": 520868352.0, + "31": 520868352.0, + "32": 520868352.0, + "33": 520868352.0, + "34": 520868352.0, + "35": 520868352.0, + "36": 520868352.0, + "37": 520868352.0, + "38": 520868352.0, + "39": 520868352.0, + "40": 520868352.0, + "41": 520868352.0, + "42": 520868352.0, + "43": 520868352.0, + "44": 520868352.0, + "45": 520868352.0, + "46": 520868352.0, + "47": 520868352.0, + "48": 520868352.0, + "49": 520868352.0, + "50": 520868352.0, + "51": 520868352.0, + "52": 520868352.0, + "53": 520868352.0, + "54": 520868352.0, + "55": 520868352.0, + "56": 520868352.0, + "57": 520868352.0, + "58": 520868352.0, + "59": 520868352.0, + "60": 520868352.0, + "61": 520868352.0, + "62": 520868352.0, + "63": 520868352.0, + "64": 520868352.0, + "65": 520868352.0, + "66": 520868352.0, + "67": 520868352.0, + "68": 520868352.0, + "69": 520868352.0, + "70": 520868352.0, + "71": 520868352.0, + "72": 520868352.0, + "73": 520868352.0, + "74": 520868352.0, + "75": 520868352.0, + "76": 520868352.0, + "77": 520868352.0, + "78": 520868352.0, + "79": 520868352.0, + "80": 520868352.0, + "81": 520868352.0, + "82": 520868352.0, + "83": 520868352.0, + "84": 520868352.0, + "85": 520868352.0, + "86": 520868352.0, + "87": 520868352.0, + "88": 520868352.0, + "89": 520868352.0, + "90": 520868352.0, + "91": 520868352.0, + "92": 520868352.0, + "93": 520868352.0, + "94": 520868352.0, + "95": 520868352.0, + "96": 520868352.0, + "97": 520868352.0, + "98": 520868352.0, + "99": 520868352.0, + "100": 520868352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2297829376.0, + "2": 2439744512.0, + "3": 2439744512.0, + "4": 2439744512.0, + "5": 2439744512.0, + "6": 2439744512.0, + "7": 2439744512.0, + "8": 2439744512.0, + "9": 2439744512.0, + "10": 2439744512.0, + "11": 2439744512.0, + "12": 2439744512.0, + "13": 2439744512.0, + "14": 2439744512.0, + "15": 2439744512.0, + "16": 2439744512.0, + "17": 2439744512.0, + "18": 2439744512.0, + "19": 2439744512.0, + "20": 2439744512.0, + "21": 2439744512.0, + "22": 2439744512.0, + "23": 2439744512.0, + "24": 2439744512.0, + "25": 2439744512.0, + "26": 2439744512.0, + "27": 2439744512.0, + "28": 2439744512.0, + "29": 2439744512.0, + "30": 2439744512.0, + "31": 2439744512.0, + "32": 2439744512.0, + "33": 2439744512.0, + "34": 2439744512.0, + "35": 2439744512.0, + "36": 2439744512.0, + "37": 2439744512.0, + "38": 2439744512.0, + "39": 2439744512.0, + "40": 2439744512.0, + "41": 2439744512.0, + "42": 2439744512.0, + "43": 2439744512.0, + "44": 2439744512.0, + "45": 2439744512.0, + "46": 2439744512.0, + "47": 2439744512.0, + "48": 2439744512.0, + "49": 2439744512.0, + "50": 2439744512.0, + "51": 2439744512.0, + "52": 2439744512.0, + "53": 2439744512.0, + "54": 2439744512.0, + "55": 2439744512.0, + "56": 2439744512.0, + "57": 2439744512.0, + "58": 2439744512.0, + "59": 2439744512.0, + "60": 2439744512.0, + "61": 2439744512.0, + "62": 2439744512.0, + "63": 2439744512.0, + "64": 2439744512.0, + "65": 2439744512.0, + "66": 2439744512.0, + "67": 2439744512.0, + "68": 2439744512.0, + "69": 2439744512.0, + "70": 2439744512.0, + "71": 2439744512.0, + "72": 2439744512.0, + "73": 2439744512.0, + "74": 2439744512.0, + "75": 2439744512.0, + "76": 2439744512.0, + "77": 2439744512.0, + "78": 2439744512.0, + "79": 2439744512.0, + "80": 2439744512.0, + "81": 2439744512.0, + "82": 2439744512.0, + "83": 2439744512.0, + "84": 2439744512.0, + "85": 2439744512.0, + "86": 2439744512.0, + "87": 2439744512.0, + "88": 2439744512.0, + "89": 2439744512.0, + "90": 2439744512.0, + "91": 2439744512.0, + "92": 2439744512.0, + "93": 2439744512.0, + "94": 2439744512.0, + "95": 2439744512.0, + "96": 2439744512.0, + "97": 2439744512.0, + "98": 2439744512.0, + "99": 2439744512.0, + "100": 2439744512.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 13.16911, + "2": 0.18867, + "3": 0.13882, + "4": 0.13669, + "5": 0.13645, + "6": 0.13563, + "7": 0.13741, + "8": 0.13506, + "9": 0.13477, + "10": 0.13539, + "11": 0.13546, + "12": 0.13513, + "13": 0.13494, + "14": 0.13581, + "15": 0.13521, + "16": 0.13419, + "17": 0.14048, + "18": 0.13853, + "19": 0.13344, + "20": 0.13345, + "21": 0.13394, + "22": 0.1343, + "23": 0.13285, + "24": 0.13319, + "25": 0.13222, + "26": 0.13344, + "27": 0.13252, + "28": 0.13239, + "29": 0.1336, + "30": 0.13332, + "31": 0.13361, + "32": 0.13239, + "33": 0.13272, + "34": 0.13285, + "35": 0.21759, + "36": 0.1331, + "37": 0.1319, + "38": 0.13293, + "39": 0.13316, + "40": 0.13306, + "41": 0.13419, + "42": 0.13323, + "43": 0.13347, + "44": 0.13313, + "45": 0.13281, + "46": 0.13654, + "47": 0.13861, + "48": 0.14195, + "49": 0.14972, + "50": 0.14396, + "51": 0.1402, + "52": 0.13309, + "53": 0.1319, + "54": 0.13165, + "55": 0.1313, + "56": 0.13223, + "57": 0.13257, + "58": 0.13107, + "59": 0.13218, + "60": 0.13223, + "61": 0.13169, + "62": 0.13257, + "63": 0.13313, + "64": 0.13312, + "65": 0.1329, + "66": 0.13333, + "67": 0.13273, + "68": 0.13284, + "69": 0.1315, + "70": 0.13199, + "71": 0.13227, + "72": 0.13324, + "73": 0.13323, + "74": 0.13184, + "75": 0.13306, + "76": 0.13246, + "77": 0.13205, + "78": 0.1314, + "79": 0.13214, + "80": 0.13251, + "81": 0.1324, + "82": 0.13223, + "83": 0.133, + "84": 0.13175, + "85": 0.1338, + "86": 0.13446, + "87": 0.13241, + "88": 0.13183, + "89": 0.13223, + "90": 0.13104, + "91": 0.13159, + "92": 0.13112, + "93": 0.13189, + "94": 0.13213, + "95": 0.13312, + "96": 0.13243, + "97": 0.21757, + "98": 0.1322, + "99": 0.13344, + "100": 0.13353 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json index 85ab75a97bf..5d20ab395ec 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69212, "25": 10.52413, "30": 10.34552, "35": 10.26241, "40": 10.07236, "45": 9.81098, "50": 9.88418, "55": 9.86374, "60": 9.47963, "65": 8.93065, "70": 9.71218, "75": 9.4005, "80": 9.39078, "85": 9.60138, "90": 9.80502, "95": 9.50821, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1621.0, "25": 1689.0, "30": 1907.0, "35": 1922.0, "40": 2045.0, "45": 2097.0, "50": 2368.0, "55": 2383.0, "60": 2509.0, "65": 2628.0, "70": 3129.0, "75": 2552.0, "80": 3279.0, "85": 3344.0, "90": 2980.0, "95": 3324.0, "100": 3467.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522989568.0, "5": 522989568.0, "10": 522989568.0, "15": 522989568.0, "20": 522989568.0, "25": 522989568.0, "30": 522989568.0, "35": 522989568.0, "40": 522989568.0, "45": 522989568.0, "50": 522989568.0, "55": 522989568.0, "60": 522989568.0, "65": 522989568.0, "70": 522989568.0, "75": 522989568.0, "80": 522989568.0, "85": 522989568.0, "90": 522989568.0, "95": 522989568.0, "100": 522989568.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3768872960.0, "5": 3911833600.0, "10": 3911833600.0, "15": 3911833600.0, "20": 3911833600.0, "25": 3911833600.0, "30": 3911833600.0, "35": 3911833600.0, "40": 3911833600.0, "45": 3911833600.0, "50": 3911833600.0, "55": 3911833600.0, "60": 3911833600.0, "65": 3911833600.0, "70": 3911833600.0, "75": 3911833600.0, "80": 3911833600.0, "85": 3911833600.0, "90": 3911833600.0, "95": 3911833600.0, "100": 3911833600.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 18.37086, "5": 0.17677, "10": 0.17771, "15": 0.1763, "20": 0.17346, "25": 0.17343, "30": 0.17441, "35": 0.17427, "40": 0.17562, "45": 0.1738, "50": 0.17384, "55": 0.17214, "60": 0.17159, "65": 0.17227, "70": 0.17173, "75": 0.17264, "80": 0.1723, "85": 0.1716, "90": 0.17085, "95": 0.17315, "100": 0.1716}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 521400832.0, + "2": 521400832.0, + "3": 521400832.0, + "4": 521400832.0, + "5": 521400832.0, + "6": 521400832.0, + "7": 521400832.0, + "8": 521400832.0, + "9": 521400832.0, + "10": 521400832.0, + "11": 521400832.0, + "12": 521400832.0, + "13": 521400832.0, + "14": 521400832.0, + "15": 521400832.0, + "16": 521400832.0, + "17": 521400832.0, + "18": 521400832.0, + "19": 521400832.0, + "20": 521400832.0, + "21": 521400832.0, + "22": 521400832.0, + "23": 521400832.0, + "24": 521400832.0, + "25": 521400832.0, + "26": 521400832.0, + "27": 521400832.0, + "28": 521400832.0, + "29": 521400832.0, + "30": 521400832.0, + "31": 521400832.0, + "32": 521400832.0, + "33": 521400832.0, + "34": 521400832.0, + "35": 521400832.0, + "36": 521400832.0, + "37": 521400832.0, + "38": 521400832.0, + "39": 521400832.0, + "40": 521400832.0, + "41": 521400832.0, + "42": 521400832.0, + "43": 521400832.0, + "44": 521400832.0, + "45": 521400832.0, + "46": 521400832.0, + "47": 521400832.0, + "48": 521400832.0, + "49": 521400832.0, + "50": 521400832.0, + "51": 521400832.0, + "52": 521400832.0, + "53": 521400832.0, + "54": 521400832.0, + "55": 521400832.0, + "56": 521400832.0, + "57": 521400832.0, + "58": 521400832.0, + "59": 521400832.0, + "60": 521400832.0, + "61": 521400832.0, + "62": 521400832.0, + "63": 521400832.0, + "64": 521400832.0, + "65": 521400832.0, + "66": 521400832.0, + "67": 521400832.0, + "68": 521400832.0, + "69": 521400832.0, + "70": 521400832.0, + "71": 521400832.0, + "72": 521400832.0, + "73": 521400832.0, + "74": 521400832.0, + "75": 521400832.0, + "76": 521400832.0, + "77": 521400832.0, + "78": 521400832.0, + "79": 521400832.0, + "80": 521400832.0, + "81": 521400832.0, + "82": 521400832.0, + "83": 521400832.0, + "84": 521400832.0, + "85": 521400832.0, + "86": 521400832.0, + "87": 521400832.0, + "88": 521400832.0, + "89": 521400832.0, + "90": 521400832.0, + "91": 521400832.0, + "92": 521400832.0, + "93": 521400832.0, + "94": 521400832.0, + "95": 521400832.0, + "96": 521400832.0, + "97": 521400832.0, + "98": 521400832.0, + "99": 521400832.0, + "100": 521400832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2297829376.0, + "2": 2439228416.0, + "3": 2439228416.0, + "4": 2439228416.0, + "5": 2439228416.0, + "6": 2439228416.0, + "7": 2439228416.0, + "8": 2439228416.0, + "9": 2439228416.0, + "10": 2439228416.0, + "11": 2439228416.0, + "12": 2439228416.0, + "13": 2439228416.0, + "14": 2439228416.0, + "15": 2439228416.0, + "16": 2439228416.0, + "17": 2439228416.0, + "18": 2439228416.0, + "19": 2439228416.0, + "20": 2439228416.0, + "21": 2439228416.0, + "22": 2439228416.0, + "23": 2439228416.0, + "24": 2439228416.0, + "25": 2439228416.0, + "26": 2439228416.0, + "27": 2439228416.0, + "28": 2439228416.0, + "29": 2439228416.0, + "30": 2439228416.0, + "31": 2439228416.0, + "32": 2439228416.0, + "33": 2439228416.0, + "34": 2439228416.0, + "35": 2439228416.0, + "36": 2439228416.0, + "37": 2439228416.0, + "38": 2439228416.0, + "39": 2439228416.0, + "40": 2439228416.0, + "41": 2439228416.0, + "42": 2439228416.0, + "43": 2439228416.0, + "44": 2439228416.0, + "45": 2439228416.0, + "46": 2439228416.0, + "47": 2439228416.0, + "48": 2439228416.0, + "49": 2439228416.0, + "50": 2439228416.0, + "51": 2439228416.0, + "52": 2439228416.0, + "53": 2439228416.0, + "54": 2439228416.0, + "55": 2439228416.0, + "56": 2439228416.0, + "57": 2439228416.0, + "58": 2439228416.0, + "59": 2439228416.0, + "60": 2439228416.0, + "61": 2439228416.0, + "62": 2439228416.0, + "63": 2439228416.0, + "64": 2439228416.0, + "65": 2439228416.0, + "66": 2439228416.0, + "67": 2439228416.0, + "68": 2439228416.0, + "69": 2439228416.0, + "70": 2439228416.0, + "71": 2439228416.0, + "72": 2439228416.0, + "73": 2439228416.0, + "74": 2439228416.0, + "75": 2439228416.0, + "76": 2439228416.0, + "77": 2439228416.0, + "78": 2439228416.0, + "79": 2439228416.0, + "80": 2439228416.0, + "81": 2439228416.0, + "82": 2439228416.0, + "83": 2439228416.0, + "84": 2439228416.0, + "85": 2439228416.0, + "86": 2439228416.0, + "87": 2439228416.0, + "88": 2439228416.0, + "89": 2439228416.0, + "90": 2439228416.0, + "91": 2439228416.0, + "92": 2439228416.0, + "93": 2439228416.0, + "94": 2439228416.0, + "95": 2439228416.0, + "96": 2439228416.0, + "97": 2439228416.0, + "98": 2439228416.0, + "99": 2439228416.0, + "100": 2439228416.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.8604, + "2": 0.16953, + "3": 0.13987, + "4": 0.13824, + "5": 0.13775, + "6": 0.13549, + "7": 0.13611, + "8": 0.13584, + "9": 0.13626, + "10": 0.13922, + "11": 0.13526, + "12": 0.13455, + "13": 0.13222, + "14": 0.13324, + "15": 0.1325, + "16": 0.13211, + "17": 0.13198, + "18": 0.13145, + "19": 0.13207, + "20": 0.13182, + "21": 0.13297, + "22": 0.1322, + "23": 0.13275, + "24": 0.1319, + "25": 0.13822, + "26": 0.13214, + "27": 0.13169, + "28": 0.13196, + "29": 0.13229, + "30": 0.13285, + "31": 0.13112, + "32": 0.13222, + "33": 0.13056, + "34": 0.13076, + "35": 0.13218, + "36": 0.13126, + "37": 0.13091, + "38": 0.13048, + "39": 0.13082, + "40": 0.1308, + "41": 0.13202, + "42": 0.1314, + "43": 0.13222, + "44": 0.13074, + "45": 0.13237, + "46": 0.13272, + "47": 0.13239, + "48": 0.13266, + "49": 0.13226, + "50": 0.13164, + "51": 0.13425, + "52": 0.13044, + "53": 0.13037, + "54": 0.13007, + "55": 0.1301, + "56": 0.13001, + "57": 0.13054, + "58": 0.12972, + "59": 0.13049, + "60": 0.13042, + "61": 0.12903, + "62": 0.13042, + "63": 0.13104, + "64": 0.13008, + "65": 0.13158, + "66": 0.13091, + "67": 0.13089, + "68": 0.13084, + "69": 0.12903, + "70": 0.13015, + "71": 0.12957, + "72": 0.12997, + "73": 0.13025, + "74": 0.12989, + "75": 0.13018, + "76": 0.12962, + "77": 0.13065, + "78": 0.12915, + "79": 0.13007, + "80": 0.12972, + "81": 0.1301, + "82": 0.12927, + "83": 0.1302, + "84": 0.12991, + "85": 0.13129, + "86": 0.13063, + "87": 0.13028, + "88": 0.1305, + "89": 0.13046, + "90": 0.12991, + "91": 0.13058, + "92": 0.13044, + "93": 0.13009, + "94": 0.1306, + "95": 0.13082, + "96": 0.13068, + "97": 0.13403, + "98": 0.13199, + "99": 0.13191, + "100": 0.13014 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..0532ff11573 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 14.21193, + "2": 0.16972, + "3": 0.14361, + "4": 0.13668, + "5": 0.13411, + "6": 0.13482, + "7": 0.13456, + "8": 0.13436, + "9": 0.13229, + "10": 0.13468, + "11": 0.29227, + "12": 0.13377, + "13": 0.13424, + "14": 0.1347, + "15": 0.13353, + "16": 0.13405, + "17": 0.13386, + "18": 0.13335, + "19": 0.1337, + "20": 0.13441, + "21": 0.28803, + "22": 0.13435, + "23": 0.13391, + "24": 0.1344, + "25": 0.13379, + "26": 0.13318, + "27": 0.13391, + "28": 0.13299, + "29": 0.13332, + "30": 0.13439, + "31": 0.28662, + "32": 0.13342, + "33": 0.1334, + "34": 0.13306, + "35": 0.13383, + "36": 0.13334, + "37": 0.1329, + "38": 0.13424, + "39": 0.13255, + "40": 0.13274, + "41": 0.28399, + "42": 0.13366, + "43": 0.13392, + "44": 0.13317, + "45": 0.13301, + "46": 0.13375, + "47": 0.13359, + "48": 0.13309, + "49": 0.13333, + "50": 0.13346, + "51": 0.28956, + "52": 0.14009, + "53": 0.13828, + "54": 0.1377, + "55": 0.13637, + "56": 0.13795, + "57": 0.13737, + "58": 0.13417, + "59": 0.13327, + "60": 0.13391, + "61": 0.29272, + "62": 0.13275, + "63": 0.13346, + "64": 0.13336, + "65": 0.13291, + "66": 0.13365, + "67": 0.13307, + "68": 0.13243, + "69": 0.1327, + "70": 0.13323, + "71": 0.28929, + "72": 0.1323, + "73": 0.13407, + "74": 0.13317, + "75": 0.132, + "76": 0.13278, + "77": 0.13359, + "78": 0.13257, + "79": 0.13226, + "80": 0.13339, + "81": 0.28457, + "82": 0.13147, + "83": 0.13289, + "84": 0.13213, + "85": 0.13375, + "86": 0.13168, + "87": 0.13351, + "88": 0.13059, + "89": 0.13189, + "90": 0.13102, + "91": 0.28755, + "92": 0.13136, + "93": 0.13235, + "94": 0.13175, + "95": 0.13072, + "96": 0.13269, + "97": 0.13234, + "98": 0.1311, + "99": 0.13293, + "100": 0.13016 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index cda47f321c8..e45c3949555 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.82005, "5": 10.85284, "10": 10.78455, "15": 10.79229, "20": 10.69212, "25": 10.52413, "30": 10.34552, "35": 10.26241, "40": 10.07236, "45": 9.81098, "50": 9.88418, "55": 9.86374, "60": 9.47963, "65": 8.93065, "70": 9.71218, "75": 9.4005, "80": 9.39078, "85": 9.60138, "90": 9.80502, "95": 9.50821, "100": 9.3924}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1559.0, "5": 1840.0, "10": 1380.0, "15": 1848.0, "20": 1621.0, "25": 1689.0, "30": 1907.0, "35": 1922.0, "40": 2045.0, "45": 2097.0, "50": 2368.0, "55": 2383.0, "60": 2509.0, "65": 2628.0, "70": 3129.0, "75": 2552.0, "80": 3279.0, "85": 3344.0, "90": 2980.0, "95": 3324.0, "100": 3467.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522989568.0, "5": 522989568.0, "10": 522989568.0, "15": 522989568.0, "20": 522989568.0, "25": 522989568.0, "30": 522989568.0, "35": 522989568.0, "40": 522989568.0, "45": 522989568.0, "50": 522989568.0, "55": 522989568.0, "60": 522989568.0, "65": 522989568.0, "70": 522989568.0, "75": 522989568.0, "80": 522989568.0, "85": 522989568.0, "90": 522989568.0, "95": 522989568.0, "100": 522989568.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3768872960.0, "5": 3911833600.0, "10": 3911833600.0, "15": 3911833600.0, "20": 3911833600.0, "25": 3911833600.0, "30": 3911833600.0, "35": 3911833600.0, "40": 3911833600.0, "45": 3911833600.0, "50": 3911833600.0, "55": 3911833600.0, "60": 3911833600.0, "65": 3911833600.0, "70": 3911833600.0, "75": 3911833600.0, "80": 3911833600.0, "85": 3911833600.0, "90": 3911833600.0, "95": 3911833600.0, "100": 3911833600.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 24.27643, "5": 0.18328, "10": 0.18099, "15": 0.17942, "20": 0.17956, "25": 0.1771, "30": 0.17799, "35": 0.17821, "40": 0.1777, "45": 0.17833, "50": 0.17766, "55": 0.17789, "60": 0.17744, "65": 0.17689, "70": 0.1767, "75": 0.17717, "80": 0.17685, "85": 0.17672, "90": 0.17477, "95": 0.17583, "100": 0.17624}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.90397, + "2": 0.16607, + "3": 0.13982, + "4": 0.14032, + "5": 0.13765, + "6": 0.13651, + "7": 0.13453, + "8": 0.13413, + "9": 0.13703, + "10": 0.13873, + "11": 0.28364, + "12": 0.13723, + "13": 0.13756, + "14": 0.1379, + "15": 0.14148, + "16": 0.1356, + "17": 0.13661, + "18": 0.13568, + "19": 0.13637, + "20": 0.1367, + "21": 0.28276, + "22": 0.13722, + "23": 0.13404, + "24": 0.13414, + "25": 0.1341, + "26": 0.13595, + "27": 0.13446, + "28": 0.13477, + "29": 0.13439, + "30": 0.13383, + "31": 0.27955, + "32": 0.13416, + "33": 0.13472, + "34": 0.13383, + "35": 0.13499, + "36": 0.13468, + "37": 0.13332, + "38": 0.13449, + "39": 0.13488, + "40": 0.1347, + "41": 0.2818, + "42": 0.13497, + "43": 0.13495, + "44": 0.13372, + "45": 0.13385, + "46": 0.13479, + "47": 0.13339, + "48": 0.13334, + "49": 0.13393, + "50": 0.13346, + "51": 0.2815, + "52": 0.13492, + "53": 0.13387, + "54": 0.13407, + "55": 0.13263, + "56": 0.13379, + "57": 0.13439, + "58": 0.13407, + "59": 0.13481, + "60": 0.13407, + "61": 0.28073, + "62": 0.13474, + "63": 0.13363, + "64": 0.13359, + "65": 0.13323, + "66": 0.13437, + "67": 0.13391, + "68": 0.13344, + "69": 0.21561, + "70": 0.1337, + "71": 0.27778, + "72": 0.13359, + "73": 0.13364, + "74": 0.13406, + "75": 0.13376, + "76": 0.13308, + "77": 0.13263, + "78": 0.13172, + "79": 0.13328, + "80": 0.13387, + "81": 0.28018, + "82": 0.13437, + "83": 0.13645, + "84": 0.13548, + "85": 0.13558, + "86": 0.13447, + "87": 0.13492, + "88": 0.13361, + "89": 0.13427, + "90": 0.13332, + "91": 0.27771, + "92": 0.13375, + "93": 0.1331, + "94": 0.13317, + "95": 0.13408, + "96": 0.13418, + "97": 0.13752, + "98": 0.13493, + "99": 0.13408, + "100": 0.13136 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json new file mode 100644 index 00000000000..6f64205e3c4 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93693, + "2": 10.92078, + "3": 10.92311, + "4": 10.908, + "5": 10.93133, + "6": 10.93476, + "7": 10.92722, + "8": 10.91987, + "9": 10.93801, + "10": 10.91377, + "11": 10.94194, + "12": 10.93222, + "13": 10.92553, + "14": 10.94301, + "15": 10.86288, + "16": 10.87945, + "17": 10.86904, + "18": 10.88527, + "19": 10.86912, + "20": 10.77336, + "21": 10.75658, + "22": 10.62975, + "23": 10.76834, + "24": 10.65017, + "25": 10.60362, + "26": 10.66269, + "27": 10.66488, + "28": 10.60369, + "29": 10.64791, + "30": 10.40493, + "31": 10.16869, + "32": 10.5139, + "33": 10.50484, + "34": 10.27115, + "35": 10.31433, + "36": 10.27029, + "37": 10.38626, + "38": 10.23175, + "39": 10.45527, + "40": 10.12357, + "41": 10.19645, + "42": 10.25476, + "43": 9.86653, + "44": 9.99586, + "45": 9.87497, + "46": 9.86189, + "47": 10.19545, + "48": 9.87912, + "49": 9.56741, + "50": 9.94519, + "51": 9.89774, + "52": 9.78773, + "53": 10.12739, + "54": 9.98265, + "55": 9.90115, + "56": 9.66568, + "57": 9.49996, + "58": 9.87635, + "59": 9.61861, + "60": 9.55292, + "61": 9.71598, + "62": 10.03451, + "63": 9.41294, + "64": 9.81949, + "65": 8.96909, + "66": 9.7478, + "67": 9.39393, + "68": 9.82085, + "69": 9.82417, + "70": 9.77915, + "71": 9.64756, + "72": 9.59555, + "73": 9.53704, + "74": 8.96643, + "75": 9.44556, + "76": 9.09922, + "77": 10.10009, + "78": 9.73576, + "79": 9.38721, + "80": 9.41961, + "81": 9.50968, + "82": 9.72303, + "83": 9.34684, + "84": 9.44855, + "85": 9.64312, + "86": 9.07818, + "87": 9.61561, + "88": 9.79072, + "89": 9.61878, + "90": 9.85929, + "91": 9.35347, + "92": 9.38799, + "93": 9.07947, + "94": 8.82329, + "95": 9.52141, + "96": 9.54598, + "97": 9.34241, + "98": 9.70512, + "99": 8.8917, + "100": 9.43443 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727688.0, + "2": 22924916.0, + "3": 22596752.0, + "4": 23219574.0, + "5": 22715594.0, + "6": 23021444.0, + "7": 22771554.0, + "8": 22926856.0, + "9": 22842096.0, + "10": 22918196.0, + "11": 22500760.0, + "12": 22460572.0, + "13": 22917018.0, + "14": 22389964.0, + "15": 22820720.0, + "16": 22830308.0, + "17": 22819122.0, + "18": 22582876.0, + "19": 22617912.0, + "20": 22693480.0, + "21": 22740412.0, + "22": 22799838.0, + "23": 22539332.0, + "24": 22770580.0, + "25": 22818910.0, + "26": 22547838.0, + "27": 22468232.0, + "28": 22452766.0, + "29": 22528906.0, + "30": 22631718.0, + "31": 22955520.0, + "32": 22584888.0, + "33": 22558558.0, + "34": 22836286.0, + "35": 22788956.0, + "36": 22589940.0, + "37": 22496912.0, + "38": 22897132.0, + "39": 22801712.0, + "40": 22657514.0, + "41": 22659596.0, + "42": 22667300.0, + "43": 22976308.0, + "44": 22745816.0, + "45": 22675136.0, + "46": 22884540.0, + "47": 22633726.0, + "48": 22928020.0, + "49": 22727656.0, + "50": 22905162.0, + "51": 22791660.0, + "52": 22748630.0, + "53": 22925420.0, + "54": 22839176.0, + "55": 22518792.0, + "56": 22877648.0, + "57": 23113592.0, + "58": 22845136.0, + "59": 22715486.0, + "60": 22743726.0, + "61": 22724104.0, + "62": 22673746.0, + "63": 22846740.0, + "64": 22823862.0, + "65": 23061360.0, + "66": 22729628.0, + "67": 22907694.0, + "68": 22609888.0, + "69": 22584610.0, + "70": 22829190.0, + "71": 22749038.0, + "72": 22655052.0, + "73": 22739796.0, + "74": 23047606.0, + "75": 23054054.0, + "76": 22901052.0, + "77": 22271520.0, + "78": 22788892.0, + "79": 22743418.0, + "80": 22706694.0, + "81": 22890786.0, + "82": 22777316.0, + "83": 22839636.0, + "84": 23010088.0, + "85": 22712948.0, + "86": 23103380.0, + "87": 22735258.0, + "88": 22636824.0, + "89": 22498828.0, + "90": 22972694.0, + "91": 22767128.0, + "92": 22808892.0, + "93": 22659368.0, + "94": 22911112.0, + "95": 23047740.0, + "96": 22828678.0, + "97": 22608174.0, + "98": 22762982.0, + "99": 22905800.0, + "100": 23015792.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.56211, + "2": 0.17032, + "3": 0.14703, + "4": 0.1399, + "5": 0.13723, + "6": 0.13231, + "7": 0.13564, + "8": 0.13449, + "9": 0.13358, + "10": 0.13307, + "11": 0.13725, + "12": 0.13165, + "13": 0.13322, + "14": 0.13288, + "15": 0.13324, + "16": 0.13281, + "17": 0.13322, + "18": 0.13237, + "19": 0.13164, + "20": 0.13184, + "21": 0.13286, + "22": 0.13291, + "23": 0.13443, + "24": 0.14138, + "25": 0.13774, + "26": 0.13805, + "27": 0.13868, + "28": 0.13766, + "29": 0.1395, + "30": 0.13538, + "31": 0.13589, + "32": 0.13529, + "33": 0.13562, + "34": 0.13525, + "35": 0.13556, + "36": 0.13544, + "37": 0.13549, + "38": 0.13569, + "39": 0.13558, + "40": 0.13561, + "41": 0.13653, + "42": 0.13569, + "43": 0.13645, + "44": 0.13586, + "45": 0.1354, + "46": 0.13606, + "47": 0.13517, + "48": 0.13615, + "49": 0.13593, + "50": 0.13488, + "51": 0.13643, + "52": 0.13115, + "53": 0.13291, + "54": 0.13115, + "55": 0.13085, + "56": 0.1309, + "57": 0.13094, + "58": 0.13092, + "59": 0.13092, + "60": 0.13094, + "61": 0.13102, + "62": 0.13092, + "63": 0.13052, + "64": 0.1313, + "65": 0.13151, + "66": 0.1314, + "67": 0.13094, + "68": 0.1308, + "69": 0.13079, + "70": 0.1309, + "71": 0.13083, + "72": 0.13083, + "73": 0.13091, + "74": 0.13061, + "75": 0.13042, + "76": 0.13145, + "77": 0.13079, + "78": 0.13108, + "79": 0.13106, + "80": 0.13055, + "81": 0.13075, + "82": 0.13043, + "83": 0.13151, + "84": 0.13173, + "85": 0.13126, + "86": 0.13187, + "87": 0.13084, + "88": 0.13098, + "89": 0.13086, + "90": 0.13054, + "91": 0.1317, + "92": 0.13098, + "93": 0.13084, + "94": 0.13088, + "95": 0.13054, + "96": 0.13148, + "97": 0.13154, + "98": 0.13179, + "99": 0.13086, + "100": 0.13121 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json index e57b1a644a0..7848ef42dd8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.9359, "5": 10.93225, "10": 10.91083, "15": 10.85723, "20": 10.77088, "25": 10.60556, "30": 10.40549, "35": 10.31366, "40": 10.12334, "45": 9.87564, "50": 9.94452, "55": 9.90094, "60": 9.55238, "65": 8.96792, "70": 9.77835, "75": 9.44605, "80": 9.42014, "85": 9.64324, "90": 9.85827, "95": 9.52085, "100": 9.43415}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 22727686.0, "5": 22715312.0, "10": 22919060.0, "15": 22821200.0, "20": 22693840.0, "25": 22819524.0, "30": 22631232.0, "35": 22787888.0, "40": 22658144.0, "45": 22674630.0, "50": 22904452.0, "55": 22519094.0, "60": 22743052.0, "65": 23061016.0, "70": 22829332.0, "75": 23054164.0, "80": 22707344.0, "85": 22712182.0, "90": 22971846.0, "95": 23047676.0, "100": 23015938.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 522900480.0, "5": 522900480.0, "10": 522900480.0, "15": 522900480.0, "20": 522900480.0, "25": 522900480.0, "30": 522900480.0, "35": 522900480.0, "40": 522900480.0, "45": 522900480.0, "50": 522900480.0, "55": 522900480.0, "60": 522900480.0, "65": 522900480.0, "70": 522900480.0, "75": 522900480.0, "80": 522900480.0, "85": 522900480.0, "90": 522900480.0, "95": 522900480.0, "100": 522900480.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 3769943040.0, "5": 3912677376.0, "10": 3912677376.0, "15": 3912677376.0, "20": 3912677376.0, "25": 3912677376.0, "30": 3912677376.0, "35": 3912677376.0, "40": 3912677376.0, "45": 3912677376.0, "50": 3912677376.0, "55": 3912677376.0, "60": 3912677376.0, "65": 3912677376.0, "70": 3912677376.0, "75": 3912677376.0, "80": 3912677376.0, "85": 3912677376.0, "90": 3912677376.0, "95": 3912677376.0, "100": 3912677376.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 22.64226, "5": 0.17675, "10": 0.17674, "15": 0.17989, "20": 0.17736, "25": 0.1752, "30": 0.17793, "35": 0.17672, "40": 0.17711, "45": 0.17664, "50": 0.17313, "55": 0.17519, "60": 0.17937, "65": 0.17909, "70": 0.17798, "75": 0.17301, "80": 0.17354, "85": 0.17313, "90": 0.1739, "95": 0.17379, "100": 0.17144}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93693, + "2": 10.92078, + "3": 10.92311, + "4": 10.908, + "5": 10.93133, + "6": 10.93476, + "7": 10.92722, + "8": 10.91987, + "9": 10.93801, + "10": 10.91377, + "11": 10.94194, + "12": 10.93222, + "13": 10.92553, + "14": 10.94301, + "15": 10.86288, + "16": 10.87945, + "17": 10.86904, + "18": 10.88527, + "19": 10.86912, + "20": 10.77336, + "21": 10.75658, + "22": 10.62975, + "23": 10.76834, + "24": 10.65017, + "25": 10.60362, + "26": 10.66269, + "27": 10.66488, + "28": 10.60369, + "29": 10.64791, + "30": 10.40493, + "31": 10.16869, + "32": 10.5139, + "33": 10.50484, + "34": 10.27115, + "35": 10.31433, + "36": 10.27029, + "37": 10.38626, + "38": 10.23175, + "39": 10.45527, + "40": 10.12357, + "41": 10.19645, + "42": 10.25476, + "43": 9.86653, + "44": 9.99586, + "45": 9.87497, + "46": 9.86189, + "47": 10.19545, + "48": 9.87912, + "49": 9.56741, + "50": 9.94519, + "51": 9.89774, + "52": 9.78773, + "53": 10.12739, + "54": 9.98265, + "55": 9.90115, + "56": 9.66568, + "57": 9.49996, + "58": 9.87635, + "59": 9.61861, + "60": 9.55292, + "61": 9.71598, + "62": 10.03451, + "63": 9.41294, + "64": 9.81949, + "65": 8.96909, + "66": 9.7478, + "67": 9.39393, + "68": 9.82085, + "69": 9.82417, + "70": 9.77915, + "71": 9.64756, + "72": 9.59555, + "73": 9.53704, + "74": 8.96643, + "75": 9.44556, + "76": 9.09922, + "77": 10.10009, + "78": 9.73576, + "79": 9.38721, + "80": 9.41961, + "81": 9.50968, + "82": 9.72303, + "83": 9.34684, + "84": 9.44855, + "85": 9.64312, + "86": 9.07818, + "87": 9.61561, + "88": 9.79072, + "89": 9.61878, + "90": 9.85929, + "91": 9.35347, + "92": 9.38799, + "93": 9.07947, + "94": 8.82329, + "95": 9.52141, + "96": 9.54598, + "97": 9.34241, + "98": 9.70512, + "99": 8.8917, + "100": 9.43443 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727688.0, + "2": 22924916.0, + "3": 22596752.0, + "4": 23219574.0, + "5": 22715594.0, + "6": 23021444.0, + "7": 22771554.0, + "8": 22926856.0, + "9": 22842096.0, + "10": 22918196.0, + "11": 22500760.0, + "12": 22460572.0, + "13": 22917018.0, + "14": 22389964.0, + "15": 22820720.0, + "16": 22830308.0, + "17": 22819122.0, + "18": 22582876.0, + "19": 22617912.0, + "20": 22693480.0, + "21": 22740412.0, + "22": 22799838.0, + "23": 22539332.0, + "24": 22770580.0, + "25": 22818910.0, + "26": 22547838.0, + "27": 22468232.0, + "28": 22452766.0, + "29": 22528906.0, + "30": 22631718.0, + "31": 22955520.0, + "32": 22584888.0, + "33": 22558558.0, + "34": 22836286.0, + "35": 22788956.0, + "36": 22589940.0, + "37": 22496912.0, + "38": 22897132.0, + "39": 22801712.0, + "40": 22657514.0, + "41": 22659596.0, + "42": 22667300.0, + "43": 22976308.0, + "44": 22745816.0, + "45": 22675136.0, + "46": 22884540.0, + "47": 22633726.0, + "48": 22928020.0, + "49": 22727656.0, + "50": 22905162.0, + "51": 22791660.0, + "52": 22748630.0, + "53": 22925420.0, + "54": 22839176.0, + "55": 22518792.0, + "56": 22877648.0, + "57": 23113592.0, + "58": 22845136.0, + "59": 22715486.0, + "60": 22743726.0, + "61": 22724104.0, + "62": 22673746.0, + "63": 22846740.0, + "64": 22823862.0, + "65": 23061360.0, + "66": 22729628.0, + "67": 22907694.0, + "68": 22609888.0, + "69": 22584610.0, + "70": 22829190.0, + "71": 22749038.0, + "72": 22655052.0, + "73": 22739796.0, + "74": 23047606.0, + "75": 23054054.0, + "76": 22901052.0, + "77": 22271520.0, + "78": 22788892.0, + "79": 22743418.0, + "80": 22706694.0, + "81": 22890786.0, + "82": 22777316.0, + "83": 22839636.0, + "84": 23010088.0, + "85": 22712948.0, + "86": 23103380.0, + "87": 22735258.0, + "88": 22636824.0, + "89": 22498828.0, + "90": 22972694.0, + "91": 22767128.0, + "92": 22808892.0, + "93": 22659368.0, + "94": 22911112.0, + "95": 23047740.0, + "96": 22828678.0, + "97": 22608174.0, + "98": 22762982.0, + "99": 22905800.0, + "100": 23015792.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 13.44016, + "2": 0.17357, + "3": 0.14155, + "4": 0.14433, + "5": 0.14312, + "6": 0.14041, + "7": 0.14082, + "8": 0.13921, + "9": 0.1399, + "10": 0.13856, + "11": 0.13995, + "12": 0.13864, + "13": 0.13803, + "14": 0.13783, + "15": 0.13752, + "16": 0.13882, + "17": 0.13834, + "18": 0.13863, + "19": 0.13872, + "20": 0.1384, + "21": 0.13424, + "22": 0.13105, + "23": 0.13094, + "24": 0.1307, + "25": 0.13252, + "26": 0.13172, + "27": 0.12995, + "28": 0.13015, + "29": 0.13002, + "30": 0.13019, + "31": 0.13071, + "32": 0.13106, + "33": 0.1305, + "34": 0.13023, + "35": 0.13178, + "36": 0.13167, + "37": 0.13002, + "38": 0.13094, + "39": 0.13093, + "40": 0.13167, + "41": 0.13178, + "42": 0.13107, + "43": 0.1328, + "44": 0.13048, + "45": 0.13046, + "46": 0.13126, + "47": 0.12901, + "48": 0.12854, + "49": 0.12862, + "50": 0.12918, + "51": 0.14204, + "52": 0.13766, + "53": 0.13573, + "54": 0.13601, + "55": 0.13392, + "56": 0.13591, + "57": 0.13683, + "58": 0.13487, + "59": 0.13645, + "60": 0.13627, + "61": 0.13507, + "62": 0.13578, + "63": 0.13619, + "64": 0.13556, + "65": 0.13673, + "66": 0.13706, + "67": 0.13535, + "68": 0.13581, + "69": 0.1342, + "70": 0.13519, + "71": 0.13563, + "72": 0.13553, + "73": 0.13626, + "74": 0.13636, + "75": 0.1351, + "76": 0.13531, + "77": 0.1341, + "78": 0.13121, + "79": 0.13164, + "80": 0.1338, + "81": 0.13214, + "82": 0.13227, + "83": 0.13301, + "84": 0.13291, + "85": 0.13384, + "86": 0.13276, + "87": 0.13499, + "88": 0.13549, + "89": 0.13554, + "90": 0.13505, + "91": 0.13486, + "92": 0.13406, + "93": 0.13522, + "94": 0.13615, + "95": 0.1365, + "96": 0.13586, + "97": 0.13623, + "98": 0.13603, + "99": 0.13615, + "100": 0.13526 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..03ebc195862 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80831, + "2": 10.81091, + "3": 10.82881, + "4": 10.82888, + "5": 10.8518, + "6": 10.84129, + "7": 10.839, + "8": 10.80345, + "9": 10.87875, + "10": 10.88064, + "11": 10.87605, + "12": 10.82786, + "13": 10.84179, + "14": 10.81686, + "15": 10.80637, + "16": 10.79763, + "17": 10.76624, + "18": 10.78069, + "19": 10.75188, + "20": 10.63371, + "21": 10.68119, + "22": 10.63819, + "23": 10.75544, + "24": 10.61129, + "25": 10.47478, + "26": 10.59702, + "27": 10.53977, + "28": 10.45162, + "29": 10.39421, + "30": 10.39348, + "31": 10.49397, + "32": 10.32158, + "33": 10.27676, + "34": 10.44629, + "35": 9.96856, + "36": 10.11335, + "37": 10.02306, + "38": 10.37579, + "39": 9.78845, + "40": 10.10124, + "41": 10.12323, + "42": 10.02969, + "43": 10.19869, + "44": 10.05528, + "45": 9.68289, + "46": 9.98143, + "47": 9.92628, + "48": 9.66986, + "49": 9.9198, + "50": 9.9226, + "51": 9.79344, + "52": 9.32777, + "53": 9.65056, + "54": 9.8608, + "55": 9.98167, + "56": 9.81701, + "57": 9.74358, + "58": 9.83, + "59": 9.32869, + "60": 9.35336, + "61": 9.45108, + "62": 10.19054, + "63": 9.35643, + "64": 9.62865, + "65": 9.70227, + "66": 9.52515, + "67": 9.6623, + "68": 9.58775, + "69": 9.38554, + "70": 9.73858, + "71": 9.87574, + "72": 9.69164, + "73": 9.39216, + "74": 9.43911, + "75": 8.95557, + "76": 9.56378, + "77": 9.6144, + "78": 9.39189, + "79": 9.52909, + "80": 9.31414, + "81": 9.70167, + "82": 9.90384, + "83": 9.31614, + "84": 9.47152, + "85": 8.97772, + "86": 9.66417, + "87": 9.43151, + "88": 9.58677, + "89": 9.52297, + "90": 9.55832, + "91": 9.62794, + "92": 9.14014, + "93": 9.42404, + "94": 9.54555, + "95": 9.13499, + "96": 8.75216, + "97": 9.58103, + "98": 9.79018, + "99": 9.37904, + "100": 9.21212 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1004.0, + "2": 1243.0, + "3": 1236.0, + "4": 1235.0, + "5": 1179.0, + "6": 1319.0, + "7": 1173.0, + "8": 1020.0, + "9": 1205.0, + "10": 1270.0, + "11": 1163.0, + "12": 1276.0, + "13": 1398.0, + "14": 1096.0, + "15": 1148.0, + "16": 1104.0, + "17": 1141.0, + "18": 1312.0, + "19": 1200.0, + "20": 1094.0, + "21": 1197.0, + "22": 1286.0, + "23": 1353.0, + "24": 1321.0, + "25": 1053.0, + "26": 1219.0, + "27": 1300.0, + "28": 1428.0, + "29": 1300.0, + "30": 1550.0, + "31": 1500.0, + "32": 1450.0, + "33": 1597.0, + "34": 1487.0, + "35": 1196.0, + "36": 1333.0, + "37": 1457.0, + "38": 1618.0, + "39": 1321.0, + "40": 1467.0, + "41": 1743.0, + "42": 1583.0, + "43": 1726.0, + "44": 1687.0, + "45": 1607.0, + "46": 1887.0, + "47": 1694.0, + "48": 1589.0, + "49": 1754.0, + "50": 1681.0, + "51": 1792.0, + "52": 1765.0, + "53": 1874.0, + "54": 1854.0, + "55": 1816.0, + "56": 1894.0, + "57": 1855.0, + "58": 1733.0, + "59": 1425.0, + "60": 1916.0, + "61": 2236.0, + "62": 2138.0, + "63": 2016.0, + "64": 2084.0, + "65": 2473.0, + "66": 2130.0, + "67": 2275.0, + "68": 2207.0, + "69": 2199.0, + "70": 2462.0, + "71": 2390.0, + "72": 2501.0, + "73": 1957.0, + "74": 2120.0, + "75": 2095.0, + "76": 2325.0, + "77": 2395.0, + "78": 2502.0, + "79": 2700.0, + "80": 1996.0, + "81": 2404.0, + "82": 2501.0, + "83": 2493.0, + "84": 2074.0, + "85": 2167.0, + "86": 2273.0, + "87": 2540.0, + "88": 2295.0, + "89": 2583.0, + "90": 2139.0, + "91": 2390.0, + "92": 1929.0, + "93": 2179.0, + "94": 2522.0, + "95": 2468.0, + "96": 2219.0, + "97": 2145.0, + "98": 2343.0, + "99": 2330.0, + "100": 1971.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2513633792.0, + "2": 2795345408.0, + "3": 2795345408.0, + "4": 2795345408.0, + "5": 2795345408.0, + "6": 2795345408.0, + "7": 2795345408.0, + "8": 2795345408.0, + "9": 2795345408.0, + "10": 2795345408.0, + "11": 2795345408.0, + "12": 2795345408.0, + "13": 2795345408.0, + "14": 2795345408.0, + "15": 2795345408.0, + "16": 2795345408.0, + "17": 2795345408.0, + "18": 2795345408.0, + "19": 2795345408.0, + "20": 2795345408.0, + "21": 2795345408.0, + "22": 2795345408.0, + "23": 2795345408.0, + "24": 2795345408.0, + "25": 2795345408.0, + "26": 2795345408.0, + "27": 2795345408.0, + "28": 2795345408.0, + "29": 2795345408.0, + "30": 2795345408.0, + "31": 2795345408.0, + "32": 2795345408.0, + "33": 2795345408.0, + "34": 2795345408.0, + "35": 2795345408.0, + "36": 2795345408.0, + "37": 2795345408.0, + "38": 2795345408.0, + "39": 2795345408.0, + "40": 2795345408.0, + "41": 2795345408.0, + "42": 2795345408.0, + "43": 2795345408.0, + "44": 2795345408.0, + "45": 2795345408.0, + "46": 2795345408.0, + "47": 2795345408.0, + "48": 2795345408.0, + "49": 2795345408.0, + "50": 2795345408.0, + "51": 2795345408.0, + "52": 2795345408.0, + "53": 2795345408.0, + "54": 2795345408.0, + "55": 2795345408.0, + "56": 2795345408.0, + "57": 2795345408.0, + "58": 2795345408.0, + "59": 2795345408.0, + "60": 2795345408.0, + "61": 2795345408.0, + "62": 2795345408.0, + "63": 2795345408.0, + "64": 2795345408.0, + "65": 2795345408.0, + "66": 2795345408.0, + "67": 2795345408.0, + "68": 2795345408.0, + "69": 2795345408.0, + "70": 2795345408.0, + "71": 2795345408.0, + "72": 2795345408.0, + "73": 2795345408.0, + "74": 2795345408.0, + "75": 2795345408.0, + "76": 2795345408.0, + "77": 2795345408.0, + "78": 2795345408.0, + "79": 2795345408.0, + "80": 2795345408.0, + "81": 2795345408.0, + "82": 2795345408.0, + "83": 2795345408.0, + "84": 2795345408.0, + "85": 2795345408.0, + "86": 2795345408.0, + "87": 2795345408.0, + "88": 2795345408.0, + "89": 2795345408.0, + "90": 2795345408.0, + "91": 2795345408.0, + "92": 2795345408.0, + "93": 2795345408.0, + "94": 2795345408.0, + "95": 2795345408.0, + "96": 2795345408.0, + "97": 2795345408.0, + "98": 2795345408.0, + "99": 2795345408.0, + "100": 2795345408.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.76623, + "2": 0.41645, + "3": 0.15729, + "4": 0.15655, + "5": 0.15439, + "6": 0.15381, + "7": 0.15343, + "8": 0.15339, + "9": 0.15322, + "10": 0.15355, + "11": 0.15239, + "12": 0.15196, + "13": 0.15235, + "14": 0.15283, + "15": 0.15313, + "16": 0.15425, + "17": 0.15411, + "18": 0.15593, + "19": 0.1533, + "20": 0.15547, + "21": 0.15314, + "22": 0.15551, + "23": 0.1542, + "24": 0.15396, + "25": 0.15334, + "26": 0.1528, + "27": 0.15224, + "28": 0.15377, + "29": 0.15378, + "30": 0.15296, + "31": 0.15277, + "32": 0.15329, + "33": 0.15349, + "34": 0.15291, + "35": 0.15336, + "36": 0.15204, + "37": 0.15238, + "38": 0.15344, + "39": 0.15283, + "40": 0.15272, + "41": 0.15285, + "42": 0.15614, + "43": 0.15406, + "44": 0.15389, + "45": 0.15591, + "46": 0.15294, + "47": 0.15419, + "48": 0.15358, + "49": 0.15525, + "50": 0.15464, + "51": 0.15896, + "52": 0.15708, + "53": 0.15757, + "54": 0.15758, + "55": 0.15641, + "56": 0.15661, + "57": 0.15804, + "58": 0.16037, + "59": 0.15723, + "60": 0.1577, + "61": 0.1593, + "62": 0.15586, + "63": 0.15828, + "64": 0.15836, + "65": 0.16023, + "66": 0.15618, + "67": 0.15583, + "68": 0.15756, + "69": 0.18281, + "70": 0.16635, + "71": 0.19055, + "72": 0.15756, + "73": 0.15973, + "74": 0.16035, + "75": 0.15764, + "76": 0.15689, + "77": 0.15696, + "78": 0.15869, + "79": 0.15855, + "80": 0.15884, + "81": 0.15812, + "82": 0.15771, + "83": 0.15837, + "84": 0.15744, + "85": 0.15771, + "86": 0.15721, + "87": 0.15692, + "88": 0.15759, + "89": 0.15908, + "90": 0.15803, + "91": 0.15786, + "92": 0.15817, + "93": 0.15819, + "94": 0.15883, + "95": 0.15933, + "96": 0.15749, + "97": 0.15871, + "98": 0.1577, + "99": 0.15695, + "100": 0.15802 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json index 3b1e19e8a67..4771e4e3c8c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.81184, "5": 10.85464, "10": 10.88256, "15": 10.80679, "20": 10.63196, "25": 10.47374, "30": 10.39285, "35": 9.96791, "40": 10.1, "45": 9.68344, "50": 9.92465, "55": 9.98132, "60": 9.3523, "65": 9.70213, "70": 9.73809, "75": 8.95616, "80": 9.31501, "85": 8.97886, "90": 9.55812, "95": 9.13529, "100": 9.21091}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1125.0, "5": 1255.0, "10": 1367.0, "15": 1127.0, "20": 1082.0, "25": 1114.0, "30": 1558.0, "35": 1292.0, "40": 1433.0, "45": 1564.0, "50": 1749.0, "55": 1718.0, "60": 1872.0, "65": 2464.0, "70": 2564.0, "75": 1884.0, "80": 1924.0, "85": 2150.0, "90": 2319.0, "95": 2518.0, "100": 2071.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 733859840.0, "5": 733859840.0, "10": 733859840.0, "15": 733859840.0, "20": 733859840.0, "25": 733859840.0, "30": 733859840.0, "35": 733859840.0, "40": 733859840.0, "45": 733859840.0, "50": 733859840.0, "55": 733859840.0, "60": 733859840.0, "65": 733859840.0, "70": 733859840.0, "75": 733859840.0, "80": 733859840.0, "85": 733859840.0, "90": 733859840.0, "95": 733859840.0, "100": 733859840.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 4312984064.0, "5": 4596792832.0, "10": 4596792832.0, "15": 4596792832.0, "20": 4596792832.0, "25": 4596792832.0, "30": 4596792832.0, "35": 4596792832.0, "40": 4596792832.0, "45": 4596792832.0, "50": 4596792832.0, "55": 4596792832.0, "60": 4596792832.0, "65": 4596792832.0, "70": 4596792832.0, "75": 4596792832.0, "80": 4596792832.0, "85": 4596792832.0, "90": 4596792832.0, "95": 4596792832.0, "100": 4596792832.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 17.99007, "5": 0.37673, "10": 0.36645, "15": 0.3643, "20": 0.36375, "25": 0.3652, "30": 0.36495, "35": 0.36435, "40": 0.36456, "45": 0.3636, "50": 0.36345, "55": 0.36651, "60": 0.36683, "65": 0.36434, "70": 0.36726, "75": 0.36517, "80": 0.56121, "85": 0.36207, "90": 0.35913, "95": 0.36112, "100": 0.36099}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80831, + "2": 10.81091, + "3": 10.82881, + "4": 10.82888, + "5": 10.8518, + "6": 10.84129, + "7": 10.839, + "8": 10.80345, + "9": 10.87875, + "10": 10.88064, + "11": 10.87605, + "12": 10.82786, + "13": 10.84179, + "14": 10.81686, + "15": 10.80637, + "16": 10.79763, + "17": 10.76624, + "18": 10.78069, + "19": 10.75188, + "20": 10.63371, + "21": 10.68119, + "22": 10.63819, + "23": 10.75544, + "24": 10.61129, + "25": 10.47478, + "26": 10.59702, + "27": 10.53977, + "28": 10.45162, + "29": 10.39421, + "30": 10.39348, + "31": 10.49397, + "32": 10.32158, + "33": 10.27676, + "34": 10.44629, + "35": 9.96856, + "36": 10.11335, + "37": 10.02306, + "38": 10.37579, + "39": 9.78845, + "40": 10.10124, + "41": 10.12323, + "42": 10.02969, + "43": 10.19869, + "44": 10.05528, + "45": 9.68289, + "46": 9.98143, + "47": 9.92628, + "48": 9.66986, + "49": 9.9198, + "50": 9.9226, + "51": 9.79344, + "52": 9.32777, + "53": 9.65056, + "54": 9.8608, + "55": 9.98167, + "56": 9.81701, + "57": 9.74358, + "58": 9.83, + "59": 9.32869, + "60": 9.35336, + "61": 9.45108, + "62": 10.19054, + "63": 9.35643, + "64": 9.62865, + "65": 9.70227, + "66": 9.52515, + "67": 9.6623, + "68": 9.58775, + "69": 9.38554, + "70": 9.73858, + "71": 9.87574, + "72": 9.69164, + "73": 9.39216, + "74": 9.43911, + "75": 8.95557, + "76": 9.56378, + "77": 9.6144, + "78": 9.39189, + "79": 9.52909, + "80": 9.31414, + "81": 9.70167, + "82": 9.90384, + "83": 9.31614, + "84": 9.47152, + "85": 8.97772, + "86": 9.66417, + "87": 9.43151, + "88": 9.58677, + "89": 9.52297, + "90": 9.55832, + "91": 9.62794, + "92": 9.14014, + "93": 9.42404, + "94": 9.54555, + "95": 9.13499, + "96": 8.75216, + "97": 9.58103, + "98": 9.79018, + "99": 9.37904, + "100": 9.21212 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1004.0, + "2": 1243.0, + "3": 1236.0, + "4": 1235.0, + "5": 1179.0, + "6": 1319.0, + "7": 1173.0, + "8": 1020.0, + "9": 1205.0, + "10": 1270.0, + "11": 1163.0, + "12": 1276.0, + "13": 1398.0, + "14": 1096.0, + "15": 1148.0, + "16": 1104.0, + "17": 1141.0, + "18": 1312.0, + "19": 1200.0, + "20": 1094.0, + "21": 1197.0, + "22": 1286.0, + "23": 1353.0, + "24": 1321.0, + "25": 1053.0, + "26": 1219.0, + "27": 1300.0, + "28": 1428.0, + "29": 1300.0, + "30": 1550.0, + "31": 1500.0, + "32": 1450.0, + "33": 1597.0, + "34": 1487.0, + "35": 1196.0, + "36": 1333.0, + "37": 1457.0, + "38": 1618.0, + "39": 1321.0, + "40": 1467.0, + "41": 1743.0, + "42": 1583.0, + "43": 1726.0, + "44": 1687.0, + "45": 1607.0, + "46": 1887.0, + "47": 1694.0, + "48": 1589.0, + "49": 1754.0, + "50": 1681.0, + "51": 1792.0, + "52": 1765.0, + "53": 1874.0, + "54": 1854.0, + "55": 1816.0, + "56": 1894.0, + "57": 1855.0, + "58": 1733.0, + "59": 1425.0, + "60": 1916.0, + "61": 2236.0, + "62": 2138.0, + "63": 2016.0, + "64": 2084.0, + "65": 2473.0, + "66": 2130.0, + "67": 2275.0, + "68": 2207.0, + "69": 2199.0, + "70": 2462.0, + "71": 2390.0, + "72": 2501.0, + "73": 1957.0, + "74": 2120.0, + "75": 2095.0, + "76": 2325.0, + "77": 2395.0, + "78": 2502.0, + "79": 2700.0, + "80": 1996.0, + "81": 2404.0, + "82": 2501.0, + "83": 2493.0, + "84": 2074.0, + "85": 2167.0, + "86": 2273.0, + "87": 2540.0, + "88": 2295.0, + "89": 2583.0, + "90": 2139.0, + "91": 2390.0, + "92": 1929.0, + "93": 2179.0, + "94": 2522.0, + "95": 2468.0, + "96": 2219.0, + "97": 2145.0, + "98": 2343.0, + "99": 2330.0, + "100": 1971.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2513633792.0, + "2": 2795345408.0, + "3": 2795345408.0, + "4": 2795345408.0, + "5": 2795345408.0, + "6": 2795345408.0, + "7": 2795345408.0, + "8": 2795345408.0, + "9": 2795345408.0, + "10": 2795345408.0, + "11": 2795345408.0, + "12": 2795345408.0, + "13": 2795345408.0, + "14": 2795345408.0, + "15": 2795345408.0, + "16": 2795345408.0, + "17": 2795345408.0, + "18": 2795345408.0, + "19": 2795345408.0, + "20": 2795345408.0, + "21": 2795345408.0, + "22": 2795345408.0, + "23": 2795345408.0, + "24": 2795345408.0, + "25": 2795345408.0, + "26": 2795345408.0, + "27": 2795345408.0, + "28": 2795345408.0, + "29": 2795345408.0, + "30": 2795345408.0, + "31": 2795345408.0, + "32": 2795345408.0, + "33": 2795345408.0, + "34": 2795345408.0, + "35": 2795345408.0, + "36": 2795345408.0, + "37": 2795345408.0, + "38": 2795345408.0, + "39": 2795345408.0, + "40": 2795345408.0, + "41": 2795345408.0, + "42": 2795345408.0, + "43": 2795345408.0, + "44": 2795345408.0, + "45": 2795345408.0, + "46": 2795345408.0, + "47": 2795345408.0, + "48": 2795345408.0, + "49": 2795345408.0, + "50": 2795345408.0, + "51": 2795345408.0, + "52": 2795345408.0, + "53": 2795345408.0, + "54": 2795345408.0, + "55": 2795345408.0, + "56": 2795345408.0, + "57": 2795345408.0, + "58": 2795345408.0, + "59": 2795345408.0, + "60": 2795345408.0, + "61": 2795345408.0, + "62": 2795345408.0, + "63": 2795345408.0, + "64": 2795345408.0, + "65": 2795345408.0, + "66": 2795345408.0, + "67": 2795345408.0, + "68": 2795345408.0, + "69": 2795345408.0, + "70": 2795345408.0, + "71": 2795345408.0, + "72": 2795345408.0, + "73": 2795345408.0, + "74": 2795345408.0, + "75": 2795345408.0, + "76": 2795345408.0, + "77": 2795345408.0, + "78": 2795345408.0, + "79": 2795345408.0, + "80": 2795345408.0, + "81": 2795345408.0, + "82": 2795345408.0, + "83": 2795345408.0, + "84": 2795345408.0, + "85": 2795345408.0, + "86": 2795345408.0, + "87": 2795345408.0, + "88": 2795345408.0, + "89": 2795345408.0, + "90": 2795345408.0, + "91": 2795345408.0, + "92": 2795345408.0, + "93": 2795345408.0, + "94": 2795345408.0, + "95": 2795345408.0, + "96": 2795345408.0, + "97": 2795345408.0, + "98": 2795345408.0, + "99": 2795345408.0, + "100": 2795345408.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.66407, + "2": 0.18828, + "3": 0.15715, + "4": 0.15685, + "5": 0.1544, + "6": 0.15356, + "7": 0.15196, + "8": 0.15101, + "9": 0.15114, + "10": 0.15067, + "11": 0.15113, + "12": 0.15109, + "13": 0.15255, + "14": 0.15181, + "15": 0.15165, + "16": 0.14989, + "17": 0.15094, + "18": 0.15062, + "19": 0.15148, + "20": 0.15014, + "21": 0.15114, + "22": 0.14973, + "23": 0.15192, + "24": 0.15003, + "25": 0.15228, + "26": 0.15066, + "27": 0.15209, + "28": 0.15056, + "29": 0.1516, + "30": 0.15083, + "31": 0.15211, + "32": 0.15028, + "33": 0.1518, + "34": 0.1494, + "35": 0.1521, + "36": 0.15002, + "37": 0.15257, + "38": 0.15095, + "39": 0.1517, + "40": 0.1501, + "41": 0.15352, + "42": 0.15453, + "43": 0.15187, + "44": 0.15281, + "45": 0.15294, + "46": 0.15214, + "47": 0.15376, + "48": 0.15363, + "49": 0.15977, + "50": 0.15249, + "51": 0.15543, + "52": 0.15363, + "53": 0.15379, + "54": 0.15555, + "55": 0.15252, + "56": 0.15295, + "57": 0.15496, + "58": 0.15756, + "59": 0.15345, + "60": 0.15784, + "61": 0.1581, + "62": 0.15302, + "63": 0.15579, + "64": 0.1536, + "65": 0.15523, + "66": 0.15593, + "67": 0.15868, + "68": 0.15303, + "69": 0.1554, + "70": 0.15409, + "71": 0.15229, + "72": 0.15299, + "73": 0.15495, + "74": 0.15601, + "75": 0.15285, + "76": 0.15774, + "77": 0.15171, + "78": 0.15423, + "79": 0.15398, + "80": 0.15445, + "81": 0.15381, + "82": 0.15311, + "83": 0.15584, + "84": 0.15556, + "85": 0.15506, + "86": 0.15314, + "87": 0.15269, + "88": 0.15515, + "89": 0.15923, + "90": 0.15325, + "91": 0.15755, + "92": 0.1543, + "93": 0.15481, + "94": 0.15321, + "95": 0.15397, + "96": 0.15322, + "97": 0.15471, + "98": 0.15631, + "99": 0.15271, + "100": 0.15653 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..cf9f4245915 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80779, + "2": 10.81028, + "3": 10.82881, + "4": 10.82931, + "5": 10.85227, + "6": 10.84112, + "7": 10.83921, + "8": 10.80433, + "9": 10.87802, + "10": 10.88031, + "11": 10.87647, + "12": 10.82798, + "13": 10.84122, + "14": 10.81649, + "15": 10.80572, + "16": 10.7979, + "17": 10.76665, + "18": 10.78004, + "19": 10.7521, + "20": 10.63378, + "21": 10.68086, + "22": 10.63834, + "23": 10.75546, + "24": 10.61158, + "25": 10.47402, + "26": 10.59699, + "27": 10.53929, + "28": 10.45199, + "29": 10.39324, + "30": 10.39328, + "31": 10.49337, + "32": 10.32183, + "33": 10.2767, + "34": 10.44619, + "35": 9.96854, + "36": 10.11277, + "37": 10.02313, + "38": 10.37525, + "39": 9.78827, + "40": 10.10093, + "41": 10.1232, + "42": 10.02925, + "43": 10.19868, + "44": 10.05532, + "45": 9.68289, + "46": 9.98153, + "47": 9.92617, + "48": 9.66965, + "49": 9.91955, + "50": 9.92258 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1058.0, + "2": 1243.0, + "3": 1255.0, + "4": 1229.0, + "5": 1178.0, + "6": 1278.0, + "7": 1129.0, + "8": 955.0, + "9": 1299.0, + "10": 1314.0, + "11": 1220.0, + "12": 1334.0, + "13": 1364.0, + "14": 1153.0, + "15": 1214.0, + "16": 1150.0, + "17": 1176.0, + "18": 1304.0, + "19": 1090.0, + "20": 1170.0, + "21": 1286.0, + "22": 1325.0, + "23": 1353.0, + "24": 1341.0, + "25": 1100.0, + "26": 1174.0, + "27": 1357.0, + "28": 1293.0, + "29": 1259.0, + "30": 1539.0, + "31": 1485.0, + "32": 1422.0, + "33": 1410.0, + "34": 1525.0, + "35": 1163.0, + "36": 1230.0, + "37": 1392.0, + "38": 1570.0, + "39": 1260.0, + "40": 1481.0, + "41": 1687.0, + "42": 1606.0, + "43": 1696.0, + "44": 1648.0, + "45": 1620.0, + "46": 1901.0, + "47": 1627.0, + "48": 1595.0, + "49": 1750.0, + "50": 1723.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4311542784.0, + "2": 4593254400.0, + "3": 4593254400.0, + "4": 4593254400.0, + "5": 4593254400.0, + "6": 4593254400.0, + "7": 4593254400.0, + "8": 4593254400.0, + "9": 4593254400.0, + "10": 4593254400.0, + "11": 4593254400.0, + "12": 4593254400.0, + "13": 4593254400.0, + "14": 4593254400.0, + "15": 4593254400.0, + "16": 4593254400.0, + "17": 4593254400.0, + "18": 4593254400.0, + "19": 4593254400.0, + "20": 4593254400.0, + "21": 4593254400.0, + "22": 4593254400.0, + "23": 4593254400.0, + "24": 4593254400.0, + "25": 4593254400.0, + "26": 4593254400.0, + "27": 4593254400.0, + "28": 4593254400.0, + "29": 4593254400.0, + "30": 4593254400.0, + "31": 4593254400.0, + "32": 4593254400.0, + "33": 4593254400.0, + "34": 4593254400.0, + "35": 4593254400.0, + "36": 4593254400.0, + "37": 4593254400.0, + "38": 4593254400.0, + "39": 4593254400.0, + "40": 4593254400.0, + "41": 4593254400.0, + "42": 4593254400.0, + "43": 4593254400.0, + "44": 4593254400.0, + "45": 4593254400.0, + "46": 4593254400.0, + "47": 4593254400.0, + "48": 4593254400.0, + "49": 4593254400.0, + "50": 4593254400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.15288, + "2": 0.50843, + "3": 0.16717, + "4": 0.16415, + "5": 0.16752, + "6": 0.16666, + "7": 0.16995, + "8": 0.17105, + "9": 0.17027, + "10": 0.17005, + "11": 0.17, + "12": 0.16945, + "13": 0.1682, + "14": 0.1687, + "15": 0.1682, + "16": 0.16899, + "17": 0.16877, + "18": 0.16877, + "19": 0.1688, + "20": 0.16832, + "21": 0.1693, + "22": 0.16953, + "23": 0.16772, + "24": 0.16919, + "25": 0.169, + "26": 0.16905, + "27": 0.16801, + "28": 0.16843, + "29": 0.1685, + "30": 0.16864, + "31": 0.16799, + "32": 0.16279, + "33": 0.16411, + "34": 0.16258, + "35": 0.16352, + "36": 0.16277, + "37": 0.16443, + "38": 0.16253, + "39": 0.16392, + "40": 0.16225, + "41": 0.16316, + "42": 0.16241, + "43": 0.16358, + "44": 0.16211, + "45": 0.16324, + "46": 0.16385, + "47": 0.16427, + "48": 0.16287, + "49": 0.16404, + "50": 0.164 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json index 0a4708ee24f..16019e9879e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.81184, "5": 10.85464, "10": 10.88256, "15": 10.80679, "20": 10.63196, "25": 10.47374, "30": 10.39285, "35": 9.96791, "40": 10.1, "45": 9.68344, "50": 9.92465}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1125.0, "5": 1255.0, "10": 1367.0, "15": 1127.0, "20": 1082.0, "25": 1114.0, "30": 1558.0, "35": 1292.0, "40": 1433.0, "45": 1564.0, "50": 1749.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 733859840.0, "5": 733859840.0, "10": 733859840.0, "15": 733859840.0, "20": 733859840.0, "25": 733859840.0, "30": 733859840.0, "35": 733859840.0, "40": 733859840.0, "45": 733859840.0, "50": 733859840.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 4312984064.0, "5": 4596792832.0, "10": 4596792832.0, "15": 4596792832.0, "20": 4596792832.0, "25": 4596792832.0, "30": 4596792832.0, "35": 4596792832.0, "40": 4596792832.0, "45": 4596792832.0, "50": 4596792832.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 15.07476, "5": 0.37209, "10": 0.36754, "15": 0.36929, "20": 0.36944, "25": 0.36826, "30": 0.3678, "35": 0.36826, "40": 0.36559, "45": 0.36731, "50": 0.36188}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80779, + "2": 10.81028, + "3": 10.82881, + "4": 10.82931, + "5": 10.85227, + "6": 10.84112, + "7": 10.83921, + "8": 10.80433, + "9": 10.87802, + "10": 10.88031, + "11": 10.87647, + "12": 10.82798, + "13": 10.84122, + "14": 10.81649, + "15": 10.80572, + "16": 10.7979, + "17": 10.76665, + "18": 10.78004, + "19": 10.7521, + "20": 10.63378, + "21": 10.68086, + "22": 10.63834, + "23": 10.75546, + "24": 10.61158, + "25": 10.47402, + "26": 10.59699, + "27": 10.53929, + "28": 10.45199, + "29": 10.39324, + "30": 10.39328, + "31": 10.49337, + "32": 10.32183, + "33": 10.2767, + "34": 10.44619, + "35": 9.96854, + "36": 10.11277, + "37": 10.02313, + "38": 10.37525, + "39": 9.78827, + "40": 10.10093, + "41": 10.1232, + "42": 10.02925, + "43": 10.19868, + "44": 10.05532, + "45": 9.68289, + "46": 9.98153, + "47": 9.92617, + "48": 9.66965, + "49": 9.91955, + "50": 9.92258 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1058.0, + "2": 1243.0, + "3": 1255.0, + "4": 1229.0, + "5": 1178.0, + "6": 1278.0, + "7": 1129.0, + "8": 955.0, + "9": 1299.0, + "10": 1314.0, + "11": 1220.0, + "12": 1334.0, + "13": 1364.0, + "14": 1153.0, + "15": 1214.0, + "16": 1150.0, + "17": 1176.0, + "18": 1304.0, + "19": 1090.0, + "20": 1170.0, + "21": 1286.0, + "22": 1325.0, + "23": 1353.0, + "24": 1341.0, + "25": 1100.0, + "26": 1174.0, + "27": 1357.0, + "28": 1293.0, + "29": 1259.0, + "30": 1539.0, + "31": 1485.0, + "32": 1422.0, + "33": 1410.0, + "34": 1525.0, + "35": 1163.0, + "36": 1230.0, + "37": 1392.0, + "38": 1570.0, + "39": 1260.0, + "40": 1481.0, + "41": 1687.0, + "42": 1606.0, + "43": 1696.0, + "44": 1648.0, + "45": 1620.0, + "46": 1901.0, + "47": 1627.0, + "48": 1595.0, + "49": 1750.0, + "50": 1723.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4311542784.0, + "2": 4593254400.0, + "3": 4593254400.0, + "4": 4593254400.0, + "5": 4593254400.0, + "6": 4593254400.0, + "7": 4593254400.0, + "8": 4593254400.0, + "9": 4593254400.0, + "10": 4593254400.0, + "11": 4593254400.0, + "12": 4593254400.0, + "13": 4593254400.0, + "14": 4593254400.0, + "15": 4593254400.0, + "16": 4593254400.0, + "17": 4593254400.0, + "18": 4593254400.0, + "19": 4593254400.0, + "20": 4593254400.0, + "21": 4593254400.0, + "22": 4593254400.0, + "23": 4593254400.0, + "24": 4593254400.0, + "25": 4593254400.0, + "26": 4593254400.0, + "27": 4593254400.0, + "28": 4593254400.0, + "29": 4593254400.0, + "30": 4593254400.0, + "31": 4593254400.0, + "32": 4593254400.0, + "33": 4593254400.0, + "34": 4593254400.0, + "35": 4593254400.0, + "36": 4593254400.0, + "37": 4593254400.0, + "38": 4593254400.0, + "39": 4593254400.0, + "40": 4593254400.0, + "41": 4593254400.0, + "42": 4593254400.0, + "43": 4593254400.0, + "44": 4593254400.0, + "45": 4593254400.0, + "46": 4593254400.0, + "47": 4593254400.0, + "48": 4593254400.0, + "49": 4593254400.0, + "50": 4593254400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.03263, + "2": 0.21266, + "3": 0.17373, + "4": 0.17827, + "5": 0.17392, + "6": 0.17641, + "7": 0.17509, + "8": 0.17211, + "9": 0.17464, + "10": 0.21373, + "11": 0.17143, + "12": 0.17137, + "13": 0.17701, + "14": 0.17242, + "15": 0.16945, + "16": 0.1686, + "17": 0.16945, + "18": 0.16793, + "19": 0.16997, + "20": 0.16992, + "21": 0.17016, + "22": 0.16832, + "23": 0.16853, + "24": 0.16912, + "25": 0.16822, + "26": 0.16908, + "27": 0.16609, + "28": 0.239, + "29": 0.16968, + "30": 0.16763, + "31": 0.16962, + "32": 0.16788, + "33": 0.1681, + "34": 0.16749, + "35": 0.16866, + "36": 0.1697, + "37": 0.16838, + "38": 0.16867, + "39": 0.16699, + "40": 0.17098, + "41": 0.1671, + "42": 0.17036, + "43": 0.16755, + "44": 0.16699, + "45": 0.1678, + "46": 0.17136, + "47": 0.16725, + "48": 0.17257, + "49": 0.16903, + "50": 0.1687 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json new file mode 100644 index 00000000000..fa4447e6f40 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.97215, + "2": 10.96419, + "3": 10.96484, + "4": 10.9625, + "5": 10.96069, + "6": 10.96314, + "7": 10.95542, + "8": 10.96012, + "9": 10.9642, + "10": 10.94938, + "11": 10.95571, + "12": 10.94758, + "13": 10.94696, + "14": 10.95407, + "15": 10.91083, + "16": 10.90945, + "17": 10.89001, + "18": 10.89837, + "19": 10.89189, + "20": 10.80856, + "21": 10.78598, + "22": 10.69678, + "23": 10.79518, + "24": 10.6983, + "25": 10.66637, + "26": 10.71173, + "27": 10.68383, + "28": 10.62344, + "29": 10.65067, + "30": 10.45487, + "31": 10.22475, + "32": 10.52171, + "33": 10.52706, + "34": 10.29181, + "35": 10.33574, + "36": 10.2871, + "37": 10.39557, + "38": 10.2536, + "39": 10.44971, + "40": 10.14791, + "41": 10.19255, + "42": 10.24716, + "43": 9.89482, + "44": 10.00764, + "45": 9.88792, + "46": 9.86452, + "47": 10.16578, + "48": 9.87929, + "49": 9.5693, + "50": 9.95204 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727298.0, + "2": 22924540.0, + "3": 22596662.0, + "4": 23219864.0, + "5": 22714604.0, + "6": 23021138.0, + "7": 22770420.0, + "8": 22926748.0, + "9": 22842456.0, + "10": 22918536.0, + "11": 22500950.0, + "12": 22459666.0, + "13": 22917172.0, + "14": 22388760.0, + "15": 22821456.0, + "16": 22831856.0, + "17": 22818966.0, + "18": 22582904.0, + "19": 22618308.0, + "20": 22695018.0, + "21": 22739412.0, + "22": 22799506.0, + "23": 22539964.0, + "24": 22771406.0, + "25": 22818944.0, + "26": 22548888.0, + "27": 22467554.0, + "28": 22453820.0, + "29": 22529468.0, + "30": 22631974.0, + "31": 22954914.0, + "32": 22584176.0, + "33": 22557850.0, + "34": 22835592.0, + "35": 22787628.0, + "36": 22589872.0, + "37": 22497168.0, + "38": 22896192.0, + "39": 22801888.0, + "40": 22658038.0, + "41": 22659420.0, + "42": 22666532.0, + "43": 22976852.0, + "44": 22746072.0, + "45": 22675348.0, + "46": 22884600.0, + "47": 22633528.0, + "48": 22929392.0, + "49": 22728496.0, + "50": 22905024.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 659815936.0, + "2": 659815936.0, + "3": 659815936.0, + "4": 659815936.0, + "5": 659815936.0, + "6": 659815936.0, + "7": 659815936.0, + "8": 659815936.0, + "9": 659815936.0, + "10": 659815936.0, + "11": 659815936.0, + "12": 659815936.0, + "13": 659815936.0, + "14": 659815936.0, + "15": 659815936.0, + "16": 659815936.0, + "17": 659815936.0, + "18": 659815936.0, + "19": 659815936.0, + "20": 659815936.0, + "21": 659815936.0, + "22": 659815936.0, + "23": 659815936.0, + "24": 659815936.0, + "25": 659815936.0, + "26": 659815936.0, + "27": 659815936.0, + "28": 659815936.0, + "29": 659815936.0, + "30": 659815936.0, + "31": 659815936.0, + "32": 659815936.0, + "33": 659815936.0, + "34": 659815936.0, + "35": 659815936.0, + "36": 659815936.0, + "37": 659815936.0, + "38": 659815936.0, + "39": 659815936.0, + "40": 659815936.0, + "41": 659815936.0, + "42": 659815936.0, + "43": 659815936.0, + "44": 659815936.0, + "45": 659815936.0, + "46": 659815936.0, + "47": 659815936.0, + "48": 659815936.0, + "49": 659815936.0, + "50": 659815936.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2128664064.0, + "2": 2387253760.0, + "3": 2387253760.0, + "4": 2387253760.0, + "5": 2387253760.0, + "6": 2387253760.0, + "7": 2387253760.0, + "8": 2387253760.0, + "9": 2387253760.0, + "10": 2387253760.0, + "11": 2387253760.0, + "12": 2387253760.0, + "13": 2387253760.0, + "14": 2387253760.0, + "15": 2387253760.0, + "16": 2387253760.0, + "17": 2387253760.0, + "18": 2387253760.0, + "19": 2387253760.0, + "20": 2387253760.0, + "21": 2387253760.0, + "22": 2387253760.0, + "23": 2387253760.0, + "24": 2387253760.0, + "25": 2387253760.0, + "26": 2387253760.0, + "27": 2387253760.0, + "28": 2387253760.0, + "29": 2387253760.0, + "30": 2387253760.0, + "31": 2387253760.0, + "32": 2387253760.0, + "33": 2387253760.0, + "34": 2387253760.0, + "35": 2387253760.0, + "36": 2387253760.0, + "37": 2387253760.0, + "38": 2387253760.0, + "39": 2387253760.0, + "40": 2387253760.0, + "41": 2387253760.0, + "42": 2387253760.0, + "43": 2387253760.0, + "44": 2387253760.0, + "45": 2387253760.0, + "46": 2387253760.0, + "47": 2387253760.0, + "48": 2387253760.0, + "49": 2387253760.0, + "50": 2387253760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.87306, + "2": 0.21422, + "3": 0.25104, + "4": 0.16762, + "5": 0.16515, + "6": 0.16439, + "7": 0.16456, + "8": 0.16608, + "9": 0.16475, + "10": 0.16561, + "11": 0.16487, + "12": 0.16315, + "13": 0.16552, + "14": 0.16522, + "15": 0.16481, + "16": 0.16526, + "17": 0.16423, + "18": 0.16401, + "19": 0.16489, + "20": 0.16413, + "21": 0.16376, + "22": 0.16471, + "23": 0.16552, + "24": 0.16391, + "25": 0.15853, + "26": 0.15805, + "27": 0.15784, + "28": 0.15821, + "29": 0.15801, + "30": 0.15776, + "31": 0.15957, + "32": 0.16205, + "33": 0.16126, + "34": 0.15977, + "35": 0.16204, + "36": 0.15979, + "37": 0.16009, + "38": 0.15888, + "39": 0.16022, + "40": 0.1597, + "41": 0.15982, + "42": 0.1593, + "43": 0.15759, + "44": 0.15811, + "45": 0.1573, + "46": 0.15807, + "47": 0.15605, + "48": 0.15694, + "49": 0.15675, + "50": 0.15757 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json index 0cd9c8700a3..775784e5ee0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.9735, - "5": 10.95594, - "10": 10.94989, - "15": 10.9115, - "20": 10.80975, - "25": 10.6662, - "30": 10.45501, - "35": 10.33418, - "40": 10.14646, - "45": 9.89112, - "50": 9.9526 + "1": 10.97215, + "2": 10.96419, + "3": 10.96484, + "4": 10.9625, + "5": 10.96069, + "6": 10.96314, + "7": 10.95542, + "8": 10.96012, + "9": 10.9642, + "10": 10.94938, + "11": 10.95571, + "12": 10.94758, + "13": 10.94696, + "14": 10.95407, + "15": 10.91083, + "16": 10.90945, + "17": 10.89001, + "18": 10.89837, + "19": 10.89189, + "20": 10.80856, + "21": 10.78598, + "22": 10.69678, + "23": 10.79518, + "24": 10.6983, + "25": 10.66637, + "26": 10.71173, + "27": 10.68383, + "28": 10.62344, + "29": 10.65067, + "30": 10.45487, + "31": 10.22475, + "32": 10.52171, + "33": 10.52706, + "34": 10.29181, + "35": 10.33574, + "36": 10.2871, + "37": 10.39557, + "38": 10.2536, + "39": 10.44971, + "40": 10.14791, + "41": 10.19255, + "42": 10.24716, + "43": 9.89482, + "44": 10.00764, + "45": 9.88792, + "46": 9.86452, + "47": 10.16578, + "48": 9.87929, + "49": 9.5693, + "50": 9.95204 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 22727052.0, - "5": 22714228.0, - "10": 22918376.0, - "15": 22820932.0, - "20": 22694228.0, - "25": 22819528.0, - "30": 22631178.0, - "35": 22787526.0, - "40": 22657932.0, - "45": 22674550.0, - "50": 22905400.0 + "1": 22727298.0, + "2": 22924540.0, + "3": 22596662.0, + "4": 23219864.0, + "5": 22714604.0, + "6": 23021138.0, + "7": 22770420.0, + "8": 22926748.0, + "9": 22842456.0, + "10": 22918536.0, + "11": 22500950.0, + "12": 22459666.0, + "13": 22917172.0, + "14": 22388760.0, + "15": 22821456.0, + "16": 22831856.0, + "17": 22818966.0, + "18": 22582904.0, + "19": 22618308.0, + "20": 22695018.0, + "21": 22739412.0, + "22": 22799506.0, + "23": 22539964.0, + "24": 22771406.0, + "25": 22818944.0, + "26": 22548888.0, + "27": 22467554.0, + "28": 22453820.0, + "29": 22529468.0, + "30": 22631974.0, + "31": 22954914.0, + "32": 22584176.0, + "33": 22557850.0, + "34": 22835592.0, + "35": 22787628.0, + "36": 22589872.0, + "37": 22497168.0, + "38": 22896192.0, + "39": 22801888.0, + "40": 22658038.0, + "41": 22659420.0, + "42": 22666532.0, + "43": 22976852.0, + "44": 22746072.0, + "45": 22675348.0, + "46": 22884600.0, + "47": 22633528.0, + "48": 22929392.0, + "49": 22728496.0, + "50": 22905024.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 657718272.0, - "5": 657718272.0, - "10": 657718272.0, - "15": 657718272.0, - "20": 657718272.0, - "25": 657718272.0, - "30": 657718272.0, - "35": 657718272.0, - "40": 657718272.0, - "45": 657718272.0, - "50": 657718272.0 + "1": 659815936.0, + "2": 659815936.0, + "3": 659815936.0, + "4": 659815936.0, + "5": 659815936.0, + "6": 659815936.0, + "7": 659815936.0, + "8": 659815936.0, + "9": 659815936.0, + "10": 659815936.0, + "11": 659815936.0, + "12": 659815936.0, + "13": 659815936.0, + "14": 659815936.0, + "15": 659815936.0, + "16": 659815936.0, + "17": 659815936.0, + "18": 659815936.0, + "19": 659815936.0, + "20": 659815936.0, + "21": 659815936.0, + "22": 659815936.0, + "23": 659815936.0, + "24": 659815936.0, + "25": 659815936.0, + "26": 659815936.0, + "27": 659815936.0, + "28": 659815936.0, + "29": 659815936.0, + "30": 659815936.0, + "31": 659815936.0, + "32": 659815936.0, + "33": 659815936.0, + "34": 659815936.0, + "35": 659815936.0, + "36": 659815936.0, + "37": 659815936.0, + "38": 659815936.0, + "39": 659815936.0, + "40": 659815936.0, + "41": 659815936.0, + "42": 659815936.0, + "43": 659815936.0, + "44": 659815936.0, + "45": 659815936.0, + "46": 659815936.0, + "47": 659815936.0, + "48": 659815936.0, + "49": 659815936.0, + "50": 659815936.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 2129712128.0, - "5": 2385156096.0, - "10": 2385156096.0, - "15": 2385156096.0, - "20": 2385156096.0, - "25": 2385156096.0, - "30": 2385156096.0, - "35": 2385156096.0, - "40": 2385156096.0, - "45": 2385156096.0, - "50": 2385156096.0 + "1": 2128664064.0, + "2": 2387253760.0, + "3": 2387253760.0, + "4": 2387253760.0, + "5": 2387253760.0, + "6": 2387253760.0, + "7": 2387253760.0, + "8": 2387253760.0, + "9": 2387253760.0, + "10": 2387253760.0, + "11": 2387253760.0, + "12": 2387253760.0, + "13": 2387253760.0, + "14": 2387253760.0, + "15": 2387253760.0, + "16": 2387253760.0, + "17": 2387253760.0, + "18": 2387253760.0, + "19": 2387253760.0, + "20": 2387253760.0, + "21": 2387253760.0, + "22": 2387253760.0, + "23": 2387253760.0, + "24": 2387253760.0, + "25": 2387253760.0, + "26": 2387253760.0, + "27": 2387253760.0, + "28": 2387253760.0, + "29": 2387253760.0, + "30": 2387253760.0, + "31": 2387253760.0, + "32": 2387253760.0, + "33": 2387253760.0, + "34": 2387253760.0, + "35": 2387253760.0, + "36": 2387253760.0, + "37": 2387253760.0, + "38": 2387253760.0, + "39": 2387253760.0, + "40": 2387253760.0, + "41": 2387253760.0, + "42": 2387253760.0, + "43": 2387253760.0, + "44": 2387253760.0, + "45": 2387253760.0, + "46": 2387253760.0, + "47": 2387253760.0, + "48": 2387253760.0, + "49": 2387253760.0, + "50": 2387253760.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 20.28193, - "5": 0.17568, - "10": 0.17422, - "15": 0.17474, - "20": 0.17247, - "25": 0.16917, - "30": 0.16924, - "35": 0.16826, - "40": 0.16896, - "45": 0.1662, - "50": 0.16732 + "1": 10.48419, + "2": 0.19482, + "3": 0.26748, + "4": 0.1633, + "5": 0.15828, + "6": 0.15656, + "7": 0.1572, + "8": 0.15759, + "9": 0.15735, + "10": 0.15751, + "11": 0.15648, + "12": 0.15605, + "13": 0.15693, + "14": 0.15672, + "15": 0.15676, + "16": 0.15664, + "17": 0.15683, + "18": 0.15646, + "19": 0.15696, + "20": 0.15623, + "21": 0.15652, + "22": 0.15759, + "23": 0.15729, + "24": 0.15687, + "25": 0.15563, + "26": 0.1575, + "27": 0.15616, + "28": 0.15855, + "29": 0.15771, + "30": 0.15851, + "31": 0.1579, + "32": 0.1587, + "33": 0.1577, + "34": 0.15827, + "35": 0.15808, + "36": 0.15825, + "37": 0.1583, + "38": 0.15836, + "39": 0.15797, + "40": 0.15829, + "41": 0.15787, + "42": 0.15789, + "43": 0.15839, + "44": 0.15862, + "45": 0.15727, + "46": 0.15919, + "47": 0.15859, + "48": 0.15898, + "49": 0.15832, + "50": 0.1586 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json new file mode 100644 index 00000000000..fc938769241 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84562, + "2": 10.8536, + "3": 10.85978, + "4": 10.84186, + "5": 10.8756, + "6": 10.88172, + "7": 10.86719, + "8": 10.84992, + "9": 10.86792, + "10": 10.83289, + "11": 10.91083, + "12": 10.87501, + "13": 10.86463, + "14": 10.8928, + "15": 10.84145, + "16": 10.84108, + "17": 10.82014, + "18": 10.85327, + "19": 10.8576, + "20": 10.80111, + "21": 10.78705, + "22": 10.72832, + "23": 10.812, + "24": 10.74372, + "25": 10.712, + "26": 10.76908, + "27": 10.78641, + "28": 10.73219, + "29": 10.75717, + "30": 10.58162, + "31": 10.43196, + "32": 10.68369, + "33": 10.66821, + "34": 10.49987, + "35": 10.5319, + "36": 10.52076, + "37": 10.59704, + "38": 10.45745, + "39": 10.62078, + "40": 10.35651, + "41": 10.40224, + "42": 10.45518, + "43": 10.11645, + "44": 10.24176, + "45": 10.1377, + "46": 10.11459, + "47": 10.39867, + "48": 10.14178, + "49": 9.8915, + "50": 10.20004 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727180.0, + "2": 22925490.0, + "3": 22595358.0, + "4": 23219094.0, + "5": 22713274.0, + "6": 23020840.0, + "7": 22770150.0, + "8": 22925912.0, + "9": 22842304.0, + "10": 22917686.0, + "11": 22499934.0, + "12": 22458902.0, + "13": 22916544.0, + "14": 22388412.0, + "15": 22820828.0, + "16": 22829442.0, + "17": 22818424.0, + "18": 22582572.0, + "19": 22616516.0, + "20": 22693786.0, + "21": 22738788.0, + "22": 22800098.0, + "23": 22538252.0, + "24": 22771304.0, + "25": 22818714.0, + "26": 22547732.0, + "27": 22467548.0, + "28": 22452916.0, + "29": 22528584.0, + "30": 22630192.0, + "31": 22954950.0, + "32": 22584400.0, + "33": 22557234.0, + "34": 22834292.0, + "35": 22786428.0, + "36": 22588576.0, + "37": 22496998.0, + "38": 22895112.0, + "39": 22800900.0, + "40": 22657380.0, + "41": 22658838.0, + "42": 22666328.0, + "43": 22975596.0, + "44": 22745924.0, + "45": 22674268.0, + "46": 22884128.0, + "47": 22632352.0, + "48": 22927496.0, + "49": 22727204.0, + "50": 22903716.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 611461632.0, + "2": 611461632.0, + "3": 611461632.0, + "4": 611461632.0, + "5": 611461632.0, + "6": 611461632.0, + "7": 611461632.0, + "8": 611461632.0, + "9": 611461632.0, + "10": 611461632.0, + "11": 611461632.0, + "12": 611461632.0, + "13": 611461632.0, + "14": 611461632.0, + "15": 611461632.0, + "16": 611461632.0, + "17": 611461632.0, + "18": 611461632.0, + "19": 611461632.0, + "20": 611461632.0, + "21": 611461632.0, + "22": 611461632.0, + "23": 611461632.0, + "24": 611461632.0, + "25": 611461632.0, + "26": 611461632.0, + "27": 611461632.0, + "28": 611461632.0, + "29": 611461632.0, + "30": 611461632.0, + "31": 611461632.0, + "32": 611461632.0, + "33": 611461632.0, + "34": 611461632.0, + "35": 611461632.0, + "36": 611461632.0, + "37": 611461632.0, + "38": 611461632.0, + "39": 611461632.0, + "40": 611461632.0, + "41": 611461632.0, + "42": 611461632.0, + "43": 611461632.0, + "44": 611461632.0, + "45": 611461632.0, + "46": 611461632.0, + "47": 611461632.0, + "48": 611461632.0, + "49": 611461632.0, + "50": 611461632.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2580665856.0, + "2": 2812987904.0, + "3": 2812987904.0, + "4": 2812987904.0, + "5": 2812987904.0, + "6": 2812987904.0, + "7": 2812987904.0, + "8": 2812987904.0, + "9": 2812987904.0, + "10": 2812987904.0, + "11": 2812987904.0, + "12": 2812987904.0, + "13": 2812987904.0, + "14": 2812987904.0, + "15": 2812987904.0, + "16": 2812987904.0, + "17": 2812987904.0, + "18": 2812987904.0, + "19": 2812987904.0, + "20": 2812987904.0, + "21": 2812987904.0, + "22": 2812987904.0, + "23": 2812987904.0, + "24": 2812987904.0, + "25": 2812987904.0, + "26": 2812987904.0, + "27": 2812987904.0, + "28": 2812987904.0, + "29": 2812987904.0, + "30": 2812987904.0, + "31": 2812987904.0, + "32": 2812987904.0, + "33": 2812987904.0, + "34": 2812987904.0, + "35": 2812987904.0, + "36": 2812987904.0, + "37": 2812987904.0, + "38": 2812987904.0, + "39": 2812987904.0, + "40": 2812987904.0, + "41": 2812987904.0, + "42": 2812987904.0, + "43": 2812987904.0, + "44": 2812987904.0, + "45": 2812987904.0, + "46": 2812987904.0, + "47": 2812987904.0, + "48": 2812987904.0, + "49": 2812987904.0, + "50": 2812987904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.97654, + "2": 0.12833, + "3": 0.09335, + "4": 0.09288, + "5": 0.09223, + "6": 0.09324, + "7": 0.09538, + "8": 0.09184, + "9": 0.09231, + "10": 0.09289, + "11": 0.09189, + "12": 0.09166, + "13": 0.09092, + "14": 0.09055, + "15": 0.09032, + "16": 0.09021, + "17": 0.09123, + "18": 0.09029, + "19": 0.09028, + "20": 0.09042, + "21": 0.08964, + "22": 0.08984, + "23": 0.08956, + "24": 0.08945, + "25": 0.09004, + "26": 0.0892, + "27": 0.08926, + "28": 0.08893, + "29": 0.08891, + "30": 0.08938, + "31": 0.08988, + "32": 0.08972, + "33": 0.08927, + "34": 0.09022, + "35": 0.0899, + "36": 0.0891, + "37": 0.08893, + "38": 0.08927, + "39": 0.08909, + "40": 0.08953, + "41": 0.08971, + "42": 0.08902, + "43": 0.09024, + "44": 0.08936, + "45": 0.0894, + "46": 0.08968, + "47": 0.08906, + "48": 0.08956, + "49": 0.08937, + "50": 0.08941 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json index 117df660805..44d53d6e9d6 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.84528, "5": 10.87387, "10": 10.83535, "15": 10.84228, "20": 10.80167, "25": 10.71149, "30": 10.58467, "35": 10.53287, "40": 10.35652, "45": 10.13839, "50": 10.20005}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 22726236.0, "5": 22714030.0, "10": 22918036.0, "15": 22821520.0, "20": 22693656.0, "25": 22819138.0, "30": 22630790.0, "35": 22787508.0, "40": 22657590.0, "45": 22674508.0, "50": 22904058.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 611461120.0, "5": 611461120.0, "10": 611461120.0, "15": 611461120.0, "20": 611461120.0, "25": 611461120.0, "30": 611461120.0, "35": 611461120.0, "40": 611461120.0, "45": 611461120.0, "50": 611461120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2582211584.0, "5": 2814533632.0, "10": 2814533632.0, "15": 2814533632.0, "20": 2814533632.0, "25": 2814533632.0, "30": 2814533632.0, "35": 2814533632.0, "40": 2814533632.0, "45": 2814533632.0, "50": 2814533632.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 14.65275, "5": 0.11877, "10": 0.11712, "15": 0.11705, "20": 0.11603, "25": 0.11576, "30": 0.11652, "35": 0.11656, "40": 0.11712, "45": 0.11593, "50": 0.11726}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84562, + "2": 10.8536, + "3": 10.85978, + "4": 10.84186, + "5": 10.8756, + "6": 10.88172, + "7": 10.86719, + "8": 10.84992, + "9": 10.86792, + "10": 10.83289, + "11": 10.91083, + "12": 10.87501, + "13": 10.86463, + "14": 10.8928, + "15": 10.84145, + "16": 10.84108, + "17": 10.82014, + "18": 10.85327, + "19": 10.8576, + "20": 10.80111, + "21": 10.78705, + "22": 10.72832, + "23": 10.812, + "24": 10.74372, + "25": 10.712, + "26": 10.76908, + "27": 10.78641, + "28": 10.73219, + "29": 10.75717, + "30": 10.58162, + "31": 10.43196, + "32": 10.68369, + "33": 10.66821, + "34": 10.49987, + "35": 10.5319, + "36": 10.52076, + "37": 10.59704, + "38": 10.45745, + "39": 10.62078, + "40": 10.35651, + "41": 10.40224, + "42": 10.45518, + "43": 10.11645, + "44": 10.24176, + "45": 10.1377, + "46": 10.11459, + "47": 10.39867, + "48": 10.14178, + "49": 9.8915, + "50": 10.20004 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727180.0, + "2": 22925490.0, + "3": 22595358.0, + "4": 23219094.0, + "5": 22713274.0, + "6": 23020840.0, + "7": 22770150.0, + "8": 22925912.0, + "9": 22842304.0, + "10": 22917686.0, + "11": 22499934.0, + "12": 22458902.0, + "13": 22916544.0, + "14": 22388412.0, + "15": 22820828.0, + "16": 22829442.0, + "17": 22818424.0, + "18": 22582572.0, + "19": 22616516.0, + "20": 22693786.0, + "21": 22738788.0, + "22": 22800098.0, + "23": 22538252.0, + "24": 22771304.0, + "25": 22818714.0, + "26": 22547732.0, + "27": 22467548.0, + "28": 22452916.0, + "29": 22528584.0, + "30": 22630192.0, + "31": 22954950.0, + "32": 22584400.0, + "33": 22557234.0, + "34": 22834292.0, + "35": 22786428.0, + "36": 22588576.0, + "37": 22496998.0, + "38": 22895112.0, + "39": 22800900.0, + "40": 22657380.0, + "41": 22658838.0, + "42": 22666328.0, + "43": 22975596.0, + "44": 22745924.0, + "45": 22674268.0, + "46": 22884128.0, + "47": 22632352.0, + "48": 22927496.0, + "49": 22727204.0, + "50": 22903716.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 609364480.0, + "2": 609364480.0, + "3": 609364480.0, + "4": 609364480.0, + "5": 609364480.0, + "6": 609364480.0, + "7": 609364480.0, + "8": 609364480.0, + "9": 609364480.0, + "10": 609364480.0, + "11": 609364480.0, + "12": 609364480.0, + "13": 609364480.0, + "14": 609364480.0, + "15": 609364480.0, + "16": 609364480.0, + "17": 609364480.0, + "18": 609364480.0, + "19": 609364480.0, + "20": 609364480.0, + "21": 609364480.0, + "22": 609364480.0, + "23": 609364480.0, + "24": 609364480.0, + "25": 609364480.0, + "26": 609364480.0, + "27": 609364480.0, + "28": 609364480.0, + "29": 609364480.0, + "30": 609364480.0, + "31": 609364480.0, + "32": 609364480.0, + "33": 609364480.0, + "34": 609364480.0, + "35": 609364480.0, + "36": 609364480.0, + "37": 609364480.0, + "38": 609364480.0, + "39": 609364480.0, + "40": 609364480.0, + "41": 609364480.0, + "42": 609364480.0, + "43": 609364480.0, + "44": 609364480.0, + "45": 609364480.0, + "46": 609364480.0, + "47": 609364480.0, + "48": 609364480.0, + "49": 609364480.0, + "50": 609364480.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2580665856.0, + "2": 2810890752.0, + "3": 2811808256.0, + "4": 2811808256.0, + "5": 2811808256.0, + "6": 2811808256.0, + "7": 2811808256.0, + "8": 2811808256.0, + "9": 2811808256.0, + "10": 2811808256.0, + "11": 2811808256.0, + "12": 2811808256.0, + "13": 2811808256.0, + "14": 2811808256.0, + "15": 2811808256.0, + "16": 2811808256.0, + "17": 2811808256.0, + "18": 2811808256.0, + "19": 2811808256.0, + "20": 2811808256.0, + "21": 2811808256.0, + "22": 2811808256.0, + "23": 2811808256.0, + "24": 2811808256.0, + "25": 2811808256.0, + "26": 2811808256.0, + "27": 2811808256.0, + "28": 2811808256.0, + "29": 2811808256.0, + "30": 2811808256.0, + "31": 2811808256.0, + "32": 2811808256.0, + "33": 2811808256.0, + "34": 2811808256.0, + "35": 2811808256.0, + "36": 2811808256.0, + "37": 2811808256.0, + "38": 2811808256.0, + "39": 2811808256.0, + "40": 2811808256.0, + "41": 2811808256.0, + "42": 2811808256.0, + "43": 2811808256.0, + "44": 2811808256.0, + "45": 2811808256.0, + "46": 2811808256.0, + "47": 2811808256.0, + "48": 2811808256.0, + "49": 2811808256.0, + "50": 2811808256.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 9.118, + "2": 0.12375, + "3": 0.31133, + "4": 0.09209, + "5": 0.09124, + "6": 0.09155, + "7": 0.09163, + "8": 0.0915, + "9": 0.09161, + "10": 0.09407, + "11": 0.09038, + "12": 0.09031, + "13": 0.09069, + "14": 0.09024, + "15": 0.09043, + "16": 0.08996, + "17": 0.09133, + "18": 0.09072, + "19": 0.09048, + "20": 0.09016, + "21": 0.09061, + "22": 0.09073, + "23": 0.09098, + "24": 0.09135, + "25": 0.09235, + "26": 0.09059, + "27": 0.09009, + "28": 0.09049, + "29": 0.09147, + "30": 0.09097, + "31": 0.09098, + "32": 0.09045, + "33": 0.09082, + "34": 0.08994, + "35": 0.09054, + "36": 0.09124, + "37": 0.09063, + "38": 0.08989, + "39": 0.09234, + "40": 0.09165, + "41": 0.09179, + "42": 0.09165, + "43": 0.09235, + "44": 0.09147, + "45": 0.0922, + "46": 0.09192, + "47": 0.09138, + "48": 0.09278, + "49": 0.09145, + "50": 0.09175 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json new file mode 100644 index 00000000000..57fe2806d3f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88765, + "2": 10.90324, + "3": 10.87803, + "4": 10.87967, + "5": 10.91013, + "6": 10.91377, + "7": 10.89285, + "8": 10.88408, + "9": 10.89749, + "10": 10.87762, + "11": 10.88984, + "12": 10.89502, + "13": 10.90738, + "14": 10.90831, + "15": 10.86285, + "16": 10.85944, + "17": 10.8221, + "18": 10.84343, + "19": 10.83663, + "20": 10.74694, + "21": 10.73044, + "22": 10.62623, + "23": 10.74379, + "24": 10.63229, + "25": 10.59789, + "26": 10.64585, + "27": 10.64828, + "28": 10.59539, + "29": 10.60615, + "30": 10.40161, + "31": 10.18116, + "32": 10.49112, + "33": 10.48508, + "34": 10.25199, + "35": 10.30784, + "36": 10.25604, + "37": 10.36576, + "38": 10.21673, + "39": 10.40989, + "40": 10.11047, + "41": 10.15883, + "42": 10.22753, + "43": 9.8745, + "44": 9.97803, + "45": 9.8591, + "46": 9.8546, + "47": 10.15264, + "48": 9.86489, + "49": 9.5555, + "50": 9.92088, + "51": 9.86094, + "52": 9.75697, + "53": 10.07633, + "54": 9.96082, + "55": 9.88565, + "56": 9.6349, + "57": 9.4925, + "58": 9.83099, + "59": 9.59122, + "60": 9.50798, + "61": 9.7061, + "62": 9.98413, + "63": 9.37604, + "64": 9.77938, + "65": 8.95852, + "66": 9.70596, + "67": 9.37402, + "68": 9.78683, + "69": 9.78932, + "70": 9.72766, + "71": 9.61135, + "72": 9.59178, + "73": 9.49896, + "74": 8.95742, + "75": 9.42469, + "76": 9.09651, + "77": 10.06653, + "78": 9.73149, + "79": 9.37959, + "80": 9.40394, + "81": 9.48277, + "82": 9.69318, + "83": 9.31104, + "84": 9.4139, + "85": 9.61469, + "86": 9.07793, + "87": 9.59662, + "88": 9.74827, + "89": 9.60196, + "90": 9.81239, + "91": 9.34524, + "92": 9.36524, + "93": 9.07745, + "94": 8.83182, + "95": 9.521, + "96": 9.52525, + "97": 9.31322, + "98": 9.677, + "99": 8.88904, + "100": 9.40063 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1753.0, + "2": 1863.0, + "3": 1781.0, + "4": 1850.0, + "5": 2021.0, + "6": 1987.0, + "7": 2006.0, + "8": 1657.0, + "9": 1961.0, + "10": 1491.0, + "11": 2010.0, + "12": 1846.0, + "13": 1934.0, + "14": 1954.0, + "15": 1942.0, + "16": 1969.0, + "17": 1936.0, + "18": 1836.0, + "19": 1841.0, + "20": 1684.0, + "21": 1958.0, + "22": 1782.0, + "23": 2112.0, + "24": 1651.0, + "25": 1713.0, + "26": 1878.0, + "27": 1837.0, + "28": 2025.0, + "29": 2115.0, + "30": 1867.0, + "31": 1622.0, + "32": 1939.0, + "33": 2167.0, + "34": 1867.0, + "35": 2035.0, + "36": 2086.0, + "37": 2477.0, + "38": 2296.0, + "39": 2505.0, + "40": 2150.0, + "41": 2315.0, + "42": 2239.0, + "43": 1941.0, + "44": 2169.0, + "45": 2172.0, + "46": 2231.0, + "47": 2604.0, + "48": 2429.0, + "49": 2289.0, + "50": 2529.0, + "51": 2742.0, + "52": 2671.0, + "53": 3066.0, + "54": 2782.0, + "55": 2510.0, + "56": 2874.0, + "57": 2304.0, + "58": 3111.0, + "59": 2862.0, + "60": 2374.0, + "61": 2977.0, + "62": 2740.0, + "63": 2394.0, + "64": 3232.0, + "65": 2720.0, + "66": 3277.0, + "67": 2810.0, + "68": 2830.0, + "69": 3094.0, + "70": 3327.0, + "71": 3106.0, + "72": 2261.0, + "73": 3147.0, + "74": 1902.0, + "75": 2545.0, + "76": 2905.0, + "77": 3468.0, + "78": 3432.0, + "79": 3336.0, + "80": 3434.0, + "81": 3605.0, + "82": 3269.0, + "83": 2891.0, + "84": 3343.0, + "85": 3501.0, + "86": 2786.0, + "87": 3872.0, + "88": 3019.0, + "89": 3407.0, + "90": 3023.0, + "91": 2630.0, + "92": 3186.0, + "93": 2746.0, + "94": 3526.0, + "95": 3414.0, + "96": 3546.0, + "97": 3339.0, + "98": 3758.0, + "99": 3058.0, + "100": 3454.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 462388736.0, + "2": 462388736.0, + "3": 462388736.0, + "4": 462388736.0, + "5": 462388736.0, + "6": 462388736.0, + "7": 462388736.0, + "8": 462388736.0, + "9": 462388736.0, + "10": 462388736.0, + "11": 462388736.0, + "12": 462388736.0, + "13": 462388736.0, + "14": 462388736.0, + "15": 462388736.0, + "16": 462388736.0, + "17": 462388736.0, + "18": 462388736.0, + "19": 462388736.0, + "20": 462388736.0, + "21": 462388736.0, + "22": 462388736.0, + "23": 462388736.0, + "24": 462388736.0, + "25": 462388736.0, + "26": 462388736.0, + "27": 462388736.0, + "28": 462388736.0, + "29": 462388736.0, + "30": 462388736.0, + "31": 462388736.0, + "32": 462388736.0, + "33": 462388736.0, + "34": 462388736.0, + "35": 462388736.0, + "36": 462388736.0, + "37": 462388736.0, + "38": 462388736.0, + "39": 462388736.0, + "40": 462388736.0, + "41": 462388736.0, + "42": 462388736.0, + "43": 462388736.0, + "44": 462388736.0, + "45": 462388736.0, + "46": 462388736.0, + "47": 462388736.0, + "48": 462388736.0, + "49": 462388736.0, + "50": 462388736.0, + "51": 462388736.0, + "52": 462388736.0, + "53": 462388736.0, + "54": 462388736.0, + "55": 462388736.0, + "56": 462388736.0, + "57": 462388736.0, + "58": 462388736.0, + "59": 462388736.0, + "60": 462388736.0, + "61": 462388736.0, + "62": 462388736.0, + "63": 462388736.0, + "64": 462388736.0, + "65": 462388736.0, + "66": 462388736.0, + "67": 462388736.0, + "68": 462388736.0, + "69": 462388736.0, + "70": 462388736.0, + "71": 462388736.0, + "72": 462388736.0, + "73": 462388736.0, + "74": 462388736.0, + "75": 462388736.0, + "76": 462388736.0, + "77": 462388736.0, + "78": 462388736.0, + "79": 462388736.0, + "80": 462388736.0, + "81": 462388736.0, + "82": 462388736.0, + "83": 462388736.0, + "84": 462388736.0, + "85": 462388736.0, + "86": 462388736.0, + "87": 462388736.0, + "88": 462388736.0, + "89": 462388736.0, + "90": 462388736.0, + "91": 462388736.0, + "92": 462388736.0, + "93": 462388736.0, + "94": 462388736.0, + "95": 462388736.0, + "96": 462388736.0, + "97": 462388736.0, + "98": 462388736.0, + "99": 462388736.0, + "100": 462388736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1549081088.0, + "2": 1677508096.0, + "3": 1677508096.0, + "4": 1677508096.0, + "5": 1677508096.0, + "6": 1677508096.0, + "7": 1677508096.0, + "8": 1677508096.0, + "9": 1677508096.0, + "10": 1677508096.0, + "11": 1677508096.0, + "12": 1677508096.0, + "13": 1677508096.0, + "14": 1677508096.0, + "15": 1677508096.0, + "16": 1677508096.0, + "17": 1677508096.0, + "18": 1677508096.0, + "19": 1677508096.0, + "20": 1677508096.0, + "21": 1677508096.0, + "22": 1677508096.0, + "23": 1677508096.0, + "24": 1677508096.0, + "25": 1677508096.0, + "26": 1677508096.0, + "27": 1677508096.0, + "28": 1677508096.0, + "29": 1677508096.0, + "30": 1677508096.0, + "31": 1677508096.0, + "32": 1677508096.0, + "33": 1677508096.0, + "34": 1677508096.0, + "35": 1677508096.0, + "36": 1677508096.0, + "37": 1677508096.0, + "38": 1677508096.0, + "39": 1677508096.0, + "40": 1677508096.0, + "41": 1677508096.0, + "42": 1677508096.0, + "43": 1677508096.0, + "44": 1677508096.0, + "45": 1677508096.0, + "46": 1677508096.0, + "47": 1677508096.0, + "48": 1677508096.0, + "49": 1677508096.0, + "50": 1677508096.0, + "51": 1677508096.0, + "52": 1677508096.0, + "53": 1677508096.0, + "54": 1677508096.0, + "55": 1677508096.0, + "56": 1677508096.0, + "57": 1677508096.0, + "58": 1677508096.0, + "59": 1677508096.0, + "60": 1677508096.0, + "61": 1677508096.0, + "62": 1677508096.0, + "63": 1677508096.0, + "64": 1677508096.0, + "65": 1677508096.0, + "66": 1677508096.0, + "67": 1677508096.0, + "68": 1677508096.0, + "69": 1677508096.0, + "70": 1677508096.0, + "71": 1677508096.0, + "72": 1677508096.0, + "73": 1677508096.0, + "74": 1677508096.0, + "75": 1677508096.0, + "76": 1677508096.0, + "77": 1677508096.0, + "78": 1677508096.0, + "79": 1677508096.0, + "80": 1677508096.0, + "81": 1677508096.0, + "82": 1677508096.0, + "83": 1677508096.0, + "84": 1677508096.0, + "85": 1677508096.0, + "86": 1677508096.0, + "87": 1677508096.0, + "88": 1677508096.0, + "89": 1677508096.0, + "90": 1677508096.0, + "91": 1677508096.0, + "92": 1677508096.0, + "93": 1677508096.0, + "94": 1677508096.0, + "95": 1677508096.0, + "96": 1677508096.0, + "97": 1677508096.0, + "98": 1677508096.0, + "99": 1677508096.0, + "100": 1677508096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.74038, + "2": 0.18941, + "3": 0.17443, + "4": 0.17279, + "5": 0.17221, + "6": 0.17285, + "7": 0.17288, + "8": 0.17181, + "9": 0.168, + "10": 0.16829, + "11": 0.16793, + "12": 0.16785, + "13": 0.16712, + "14": 0.17584, + "15": 0.16768, + "16": 0.16694, + "17": 0.16606, + "18": 0.16539, + "19": 0.16505, + "20": 0.16548, + "21": 0.16486, + "22": 0.16481, + "23": 0.1694, + "24": 0.16549, + "25": 0.17312, + "26": 0.16555, + "27": 0.17362, + "28": 0.16604, + "29": 0.1708, + "30": 0.16752, + "31": 0.16804, + "32": 0.16714, + "33": 0.16525, + "34": 0.16728, + "35": 0.16428, + "36": 0.17043, + "37": 0.16532, + "38": 0.16525, + "39": 0.16555, + "40": 0.16531, + "41": 0.16563, + "42": 0.16563, + "43": 0.16499, + "44": 0.16632, + "45": 0.16537, + "46": 0.16563, + "47": 0.16529, + "48": 0.1657, + "49": 0.16462, + "50": 0.16554, + "51": 0.17653, + "52": 0.17191, + "53": 0.17217, + "54": 0.17273, + "55": 0.17175, + "56": 0.1728, + "57": 0.17152, + "58": 0.17298, + "59": 0.17049, + "60": 0.16917, + "61": 0.16777, + "62": 0.16726, + "63": 0.16773, + "64": 0.16713, + "65": 0.16836, + "66": 0.1678, + "67": 0.1679, + "68": 0.16772, + "69": 0.16815, + "70": 0.16816, + "71": 0.16728, + "72": 0.16778, + "73": 0.16743, + "74": 0.16783, + "75": 0.16824, + "76": 0.16759, + "77": 0.16828, + "78": 0.16749, + "79": 0.16757, + "80": 0.16781, + "81": 0.16773, + "82": 0.16701, + "83": 0.16756, + "84": 0.16842, + "85": 0.16707, + "86": 0.16752, + "87": 0.16807, + "88": 0.17901, + "89": 0.17301, + "90": 0.1702, + "91": 0.17493, + "92": 0.17105, + "93": 0.16969, + "94": 0.16931, + "95": 0.17031, + "96": 0.16969, + "97": 0.17032, + "98": 0.17059, + "99": 0.17, + "100": 0.17572 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json index 053ded61519..3be93706d81 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.88734, "5": 10.90948, "10": 10.87763, "15": 10.86371, "20": 10.7509, "25": 10.59917, "30": 10.40104, "35": 10.30793, "40": 10.10902, "45": 9.85831, "50": 9.92111, "55": 9.88529, "60": 9.50737, "65": 8.95828, "70": 9.72733, "75": 9.42571, "80": 9.40559, "85": 9.61572, "90": 9.81277, "95": 9.52119, "100": 9.40111}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1614.0, "5": 2021.0, "10": 1399.0, "15": 1872.0, "20": 1692.0, "25": 1784.0, "30": 1900.0, "35": 2081.0, "40": 2233.0, "45": 2144.0, "50": 2513.0, "55": 2493.0, "60": 2440.0, "65": 2763.0, "70": 3219.0, "75": 2601.0, "80": 3341.0, "85": 3433.0, "90": 3019.0, "95": 3417.0, "100": 3372.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 462455808.0, "5": 462455808.0, "10": 462455808.0, "15": 462455808.0, "20": 462455808.0, "25": 462455808.0, "30": 462455808.0, "35": 462455808.0, "40": 462455808.0, "45": 462455808.0, "50": 462455808.0, "55": 462455808.0, "60": 462455808.0, "65": 462455808.0, "70": 462455808.0, "75": 462455808.0, "80": 462455808.0, "85": 462455808.0, "90": 462455808.0, "95": 462455808.0, "100": 462455808.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2529822720.0, "5": 2658249728.0, "10": 2658249728.0, "15": 2658249728.0, "20": 2658249728.0, "25": 2658249728.0, "30": 2658249728.0, "35": 2658249728.0, "40": 2658249728.0, "45": 2658249728.0, "50": 2658249728.0, "55": 2658249728.0, "60": 2658249728.0, "65": 2658249728.0, "70": 2658249728.0, "75": 2658249728.0, "80": 2658249728.0, "85": 2658249728.0, "90": 2658249728.0, "95": 2658249728.0, "100": 2658249728.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 11.76278, "5": 0.18711, "10": 0.18109, "15": 0.18221, "20": 0.18082, "25": 0.18031, "30": 0.18054, "35": 0.17683, "40": 0.17853, "45": 0.17695, "50": 0.17851, "55": 0.1773, "60": 0.17609, "65": 0.18072, "70": 0.18082, "75": 0.17652, "80": 0.17661, "85": 0.18002, "90": 0.17586, "95": 0.17539, "100": 0.17485}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88765, + "2": 10.90324, + "3": 10.87803, + "4": 10.87967, + "5": 10.91013, + "6": 10.91377, + "7": 10.89285, + "8": 10.88408, + "9": 10.89749, + "10": 10.87762, + "11": 10.88984, + "12": 10.89502, + "13": 10.90738, + "14": 10.90831, + "15": 10.86285, + "16": 10.85944, + "17": 10.8221, + "18": 10.84343, + "19": 10.83663, + "20": 10.74694, + "21": 10.73044, + "22": 10.62623, + "23": 10.74379, + "24": 10.63229, + "25": 10.59789, + "26": 10.64585, + "27": 10.64828, + "28": 10.59539, + "29": 10.60615, + "30": 10.40161, + "31": 10.18116, + "32": 10.49112, + "33": 10.48508, + "34": 10.25199, + "35": 10.30784, + "36": 10.25604, + "37": 10.36576, + "38": 10.21673, + "39": 10.40989, + "40": 10.11047, + "41": 10.15883, + "42": 10.22753, + "43": 9.8745, + "44": 9.97803, + "45": 9.8591, + "46": 9.8546, + "47": 10.15264, + "48": 9.86489, + "49": 9.5555, + "50": 9.92088, + "51": 9.86094, + "52": 9.75697, + "53": 10.07633, + "54": 9.96082, + "55": 9.88565, + "56": 9.6349, + "57": 9.4925, + "58": 9.83099, + "59": 9.59122, + "60": 9.50798, + "61": 9.7061, + "62": 9.98413, + "63": 9.37604, + "64": 9.77938, + "65": 8.95852, + "66": 9.70596, + "67": 9.37402, + "68": 9.78683, + "69": 9.78932, + "70": 9.72766, + "71": 9.61135, + "72": 9.59178, + "73": 9.49896, + "74": 8.95742, + "75": 9.42469, + "76": 9.09651, + "77": 10.06653, + "78": 9.73149, + "79": 9.37959, + "80": 9.40394, + "81": 9.48277, + "82": 9.69318, + "83": 9.31104, + "84": 9.4139, + "85": 9.61469, + "86": 9.07793, + "87": 9.59662, + "88": 9.74827, + "89": 9.60196, + "90": 9.81239, + "91": 9.34524, + "92": 9.36524, + "93": 9.07745, + "94": 8.83182, + "95": 9.521, + "96": 9.52525, + "97": 9.31322, + "98": 9.677, + "99": 8.88904, + "100": 9.40063 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1753.0, + "2": 1863.0, + "3": 1781.0, + "4": 1850.0, + "5": 2021.0, + "6": 1987.0, + "7": 2006.0, + "8": 1657.0, + "9": 1961.0, + "10": 1491.0, + "11": 2010.0, + "12": 1846.0, + "13": 1934.0, + "14": 1954.0, + "15": 1942.0, + "16": 1969.0, + "17": 1936.0, + "18": 1836.0, + "19": 1841.0, + "20": 1684.0, + "21": 1958.0, + "22": 1782.0, + "23": 2112.0, + "24": 1651.0, + "25": 1713.0, + "26": 1878.0, + "27": 1837.0, + "28": 2025.0, + "29": 2115.0, + "30": 1867.0, + "31": 1622.0, + "32": 1939.0, + "33": 2167.0, + "34": 1867.0, + "35": 2035.0, + "36": 2086.0, + "37": 2477.0, + "38": 2296.0, + "39": 2505.0, + "40": 2150.0, + "41": 2315.0, + "42": 2239.0, + "43": 1941.0, + "44": 2169.0, + "45": 2172.0, + "46": 2231.0, + "47": 2604.0, + "48": 2429.0, + "49": 2289.0, + "50": 2529.0, + "51": 2742.0, + "52": 2671.0, + "53": 3066.0, + "54": 2782.0, + "55": 2510.0, + "56": 2874.0, + "57": 2304.0, + "58": 3111.0, + "59": 2862.0, + "60": 2374.0, + "61": 2977.0, + "62": 2740.0, + "63": 2394.0, + "64": 3232.0, + "65": 2720.0, + "66": 3277.0, + "67": 2810.0, + "68": 2830.0, + "69": 3094.0, + "70": 3327.0, + "71": 3106.0, + "72": 2261.0, + "73": 3147.0, + "74": 1902.0, + "75": 2545.0, + "76": 2905.0, + "77": 3468.0, + "78": 3432.0, + "79": 3336.0, + "80": 3434.0, + "81": 3605.0, + "82": 3269.0, + "83": 2891.0, + "84": 3343.0, + "85": 3501.0, + "86": 2786.0, + "87": 3872.0, + "88": 3019.0, + "89": 3407.0, + "90": 3023.0, + "91": 2630.0, + "92": 3186.0, + "93": 2746.0, + "94": 3526.0, + "95": 3414.0, + "96": 3546.0, + "97": 3339.0, + "98": 3758.0, + "99": 3058.0, + "100": 3454.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 462388736.0, + "2": 462388736.0, + "3": 462388736.0, + "4": 462388736.0, + "5": 462388736.0, + "6": 462388736.0, + "7": 462388736.0, + "8": 462388736.0, + "9": 462388736.0, + "10": 462388736.0, + "11": 462388736.0, + "12": 462388736.0, + "13": 462388736.0, + "14": 462388736.0, + "15": 462388736.0, + "16": 462388736.0, + "17": 462388736.0, + "18": 462388736.0, + "19": 462388736.0, + "20": 462388736.0, + "21": 462388736.0, + "22": 462388736.0, + "23": 462388736.0, + "24": 462388736.0, + "25": 462388736.0, + "26": 462388736.0, + "27": 462388736.0, + "28": 462388736.0, + "29": 462388736.0, + "30": 462388736.0, + "31": 462388736.0, + "32": 462388736.0, + "33": 462388736.0, + "34": 462388736.0, + "35": 462388736.0, + "36": 462388736.0, + "37": 462388736.0, + "38": 462388736.0, + "39": 462388736.0, + "40": 462388736.0, + "41": 462388736.0, + "42": 462388736.0, + "43": 462388736.0, + "44": 462388736.0, + "45": 462388736.0, + "46": 462388736.0, + "47": 462388736.0, + "48": 462388736.0, + "49": 462388736.0, + "50": 462388736.0, + "51": 462388736.0, + "52": 462388736.0, + "53": 462388736.0, + "54": 462388736.0, + "55": 462388736.0, + "56": 462388736.0, + "57": 462388736.0, + "58": 462388736.0, + "59": 462388736.0, + "60": 462388736.0, + "61": 462388736.0, + "62": 462388736.0, + "63": 462388736.0, + "64": 462388736.0, + "65": 462388736.0, + "66": 462388736.0, + "67": 462388736.0, + "68": 462388736.0, + "69": 462388736.0, + "70": 462388736.0, + "71": 462388736.0, + "72": 462388736.0, + "73": 462388736.0, + "74": 462388736.0, + "75": 462388736.0, + "76": 462388736.0, + "77": 462388736.0, + "78": 462388736.0, + "79": 462388736.0, + "80": 462388736.0, + "81": 462388736.0, + "82": 462388736.0, + "83": 462388736.0, + "84": 462388736.0, + "85": 462388736.0, + "86": 462388736.0, + "87": 462388736.0, + "88": 462388736.0, + "89": 462388736.0, + "90": 462388736.0, + "91": 462388736.0, + "92": 462388736.0, + "93": 462388736.0, + "94": 462388736.0, + "95": 462388736.0, + "96": 462388736.0, + "97": 462388736.0, + "98": 462388736.0, + "99": 462388736.0, + "100": 462388736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1549081088.0, + "2": 1677508096.0, + "3": 1677508096.0, + "4": 1677508096.0, + "5": 1677508096.0, + "6": 1677508096.0, + "7": 1677508096.0, + "8": 1677508096.0, + "9": 1677508096.0, + "10": 1677508096.0, + "11": 1677508096.0, + "12": 1677508096.0, + "13": 1677508096.0, + "14": 1677508096.0, + "15": 1677508096.0, + "16": 1677508096.0, + "17": 1677508096.0, + "18": 1677508096.0, + "19": 1677508096.0, + "20": 1677508096.0, + "21": 1677508096.0, + "22": 1677508096.0, + "23": 1677508096.0, + "24": 1677508096.0, + "25": 1677508096.0, + "26": 1677508096.0, + "27": 1677508096.0, + "28": 1677508096.0, + "29": 1677508096.0, + "30": 1677508096.0, + "31": 1677508096.0, + "32": 1677508096.0, + "33": 1677508096.0, + "34": 1677508096.0, + "35": 1677508096.0, + "36": 1677508096.0, + "37": 1677508096.0, + "38": 1677508096.0, + "39": 1677508096.0, + "40": 1677508096.0, + "41": 1677508096.0, + "42": 1677508096.0, + "43": 1677508096.0, + "44": 1677508096.0, + "45": 1677508096.0, + "46": 1677508096.0, + "47": 1677508096.0, + "48": 1677508096.0, + "49": 1677508096.0, + "50": 1677508096.0, + "51": 1677508096.0, + "52": 1677508096.0, + "53": 1677508096.0, + "54": 1677508096.0, + "55": 1677508096.0, + "56": 1677508096.0, + "57": 1677508096.0, + "58": 1677508096.0, + "59": 1677508096.0, + "60": 1677508096.0, + "61": 1677508096.0, + "62": 1677508096.0, + "63": 1677508096.0, + "64": 1677508096.0, + "65": 1677508096.0, + "66": 1677508096.0, + "67": 1677508096.0, + "68": 1677508096.0, + "69": 1677508096.0, + "70": 1677508096.0, + "71": 1677508096.0, + "72": 1677508096.0, + "73": 1677508096.0, + "74": 1677508096.0, + "75": 1677508096.0, + "76": 1677508096.0, + "77": 1677508096.0, + "78": 1677508096.0, + "79": 1677508096.0, + "80": 1677508096.0, + "81": 1677508096.0, + "82": 1677508096.0, + "83": 1677508096.0, + "84": 1677508096.0, + "85": 1677508096.0, + "86": 1677508096.0, + "87": 1677508096.0, + "88": 1677508096.0, + "89": 1677508096.0, + "90": 1677508096.0, + "91": 1677508096.0, + "92": 1677508096.0, + "93": 1677508096.0, + "94": 1677508096.0, + "95": 1677508096.0, + "96": 1677508096.0, + "97": 1677508096.0, + "98": 1677508096.0, + "99": 1677508096.0, + "100": 1677508096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.17153, + "2": 0.2103, + "3": 0.21541, + "4": 0.21948, + "5": 0.17282, + "6": 0.16921, + "7": 0.1711, + "8": 0.16967, + "9": 0.17064, + "10": 0.16972, + "11": 0.1696, + "12": 0.1701, + "13": 0.16923, + "14": 0.16942, + "15": 0.16782, + "16": 0.17, + "17": 0.16748, + "18": 0.16821, + "19": 0.16739, + "20": 0.16883, + "21": 0.16894, + "22": 0.16847, + "23": 0.16846, + "24": 0.16887, + "25": 0.16905, + "26": 0.16873, + "27": 0.16876, + "28": 0.16868, + "29": 0.1706, + "30": 0.17379, + "31": 0.17109, + "32": 0.17107, + "33": 0.17072, + "34": 0.17137, + "35": 0.17105, + "36": 0.17106, + "37": 0.17077, + "38": 0.17115, + "39": 0.17067, + "40": 0.17057, + "41": 0.17099, + "42": 0.17074, + "43": 0.17091, + "44": 0.17078, + "45": 0.17104, + "46": 0.17055, + "47": 0.17137, + "48": 0.17086, + "49": 0.17081, + "50": 0.17053, + "51": 0.17448, + "52": 0.16607, + "53": 0.16686, + "54": 0.16608, + "55": 0.16654, + "56": 0.16591, + "57": 0.16614, + "58": 0.1659, + "59": 0.16577, + "60": 0.16589, + "61": 0.16557, + "62": 0.16528, + "63": 0.16612, + "64": 0.1658, + "65": 0.16543, + "66": 0.1651, + "67": 0.16559, + "68": 0.16502, + "69": 0.16533, + "70": 0.16636, + "71": 0.16516, + "72": 0.1657, + "73": 0.1656, + "74": 0.16521, + "75": 0.16623, + "76": 0.16628, + "77": 0.16593, + "78": 0.16615, + "79": 0.1658, + "80": 0.16904, + "81": 0.16665, + "82": 0.16575, + "83": 0.16623, + "84": 0.16603, + "85": 0.16577, + "86": 0.16568, + "87": 0.16525, + "88": 0.16531, + "89": 0.16616, + "90": 0.16544, + "91": 0.16581, + "92": 0.16545, + "93": 0.16603, + "94": 0.16501, + "95": 0.16632, + "96": 0.16545, + "97": 0.16577, + "98": 0.19996, + "99": 0.19154, + "100": 0.19156 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..1cf5408b8ba --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.7942, + "2": 0.22889, + "3": 0.19794, + "4": 0.19876, + "5": 0.19802, + "6": 0.19732, + "7": 0.19829, + "8": 0.19869, + "9": 0.20221, + "10": 0.19735, + "11": 0.1983, + "12": 0.19776, + "13": 0.19772, + "14": 0.19813, + "15": 0.20156, + "16": 0.19951, + "17": 0.19694, + "18": 0.19906, + "19": 0.19681, + "20": 0.19852, + "21": 0.20083, + "22": 0.19703, + "23": 0.19864, + "24": 0.1981, + "25": 0.19771, + "26": 0.19763, + "27": 0.19879, + "28": 0.20245, + "29": 0.19781, + "30": 0.19864, + "31": 0.19901, + "32": 0.19832, + "33": 0.19887, + "34": 0.20272, + "35": 0.19918, + "36": 0.19934, + "37": 0.19832, + "38": 0.19851, + "39": 0.19719, + "40": 0.20496, + "41": 0.19802, + "42": 0.19778, + "43": 0.19799, + "44": 0.19797, + "45": 0.21191, + "46": 0.20272, + "47": 0.20241, + "48": 0.202, + "49": 0.20032, + "50": 0.20198, + "51": 0.21717, + "52": 0.21085, + "53": 0.20963, + "54": 0.20989, + "55": 0.20877, + "56": 0.20983, + "57": 0.21317, + "58": 0.21422, + "59": 0.2112, + "60": 0.21308, + "61": 0.21283, + "62": 0.21356, + "63": 0.21093, + "64": 0.20492, + "65": 0.20132, + "66": 0.20054, + "67": 0.19914, + "68": 0.19918, + "69": 0.19966, + "70": 0.19998, + "71": 0.19998, + "72": 0.19839, + "73": 0.19973, + "74": 0.19856, + "75": 0.19805, + "76": 0.19968, + "77": 0.19757, + "78": 0.19789, + "79": 0.19675, + "80": 0.1975, + "81": 0.19765, + "82": 0.19742, + "83": 0.19785, + "84": 0.19665, + "85": 0.19764, + "86": 0.19704, + "87": 0.19705, + "88": 0.19866, + "89": 0.19749, + "90": 0.1966, + "91": 0.19783, + "92": 0.19848, + "93": 0.19735, + "94": 0.19683, + "95": 0.19659, + "96": 0.19746, + "97": 0.19792, + "98": 0.19802, + "99": 0.19675, + "100": 0.19692 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json index b47c82b17f0..16e4a038563 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.92705, - "5": 10.92795, - "10": 10.90786, - "15": 10.88314, - "20": 10.77629, - "25": 10.59141, - "30": 10.39192, - "35": 10.29686, - "40": 10.0964, - "45": 9.84464, - "50": 9.90918, - "55": 9.87762, - "60": 9.49121, - "65": 8.94236, - "70": 9.72266, - "75": 9.41912, - "80": 9.40077, - "85": 9.61207, - "90": 9.81015, - "95": 9.51716, - "100": 9.4015 + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1627.0, - "5": 2010.0, - "10": 1368.0, - "15": 1897.0, - "20": 1626.0, - "25": 1743.0, - "30": 1930.0, - "35": 1954.0, - "40": 2199.0, - "45": 2068.0, - "50": 2460.0, - "55": 2427.0, - "60": 2380.0, - "65": 2657.0, - "70": 3265.0, - "75": 2675.0, - "80": 3434.0, - "85": 3302.0, - "90": 3230.0, - "95": 3340.0, - "100": 3291.0 + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 487096320.0, - "5": 487096320.0, - "10": 487096320.0, - "15": 487096320.0, - "20": 487096320.0, - "25": 487096320.0, - "30": 487096320.0, - "35": 487096320.0, - "40": 487096320.0, - "45": 487096320.0, - "50": 487096320.0, - "55": 487096320.0, - "60": 487096320.0, - "65": 487096320.0, - "70": 487096320.0, - "75": 487096320.0, - "80": 487096320.0, - "85": 487096320.0, - "90": 487096320.0, - "95": 487096320.0, - "100": 487096320.0 + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1719035904.0, - "5": 1900157952.0, - "10": 1901729792.0, - "15": 1901729792.0, - "20": 1901729792.0, - "25": 1901729792.0, - "30": 1901729792.0, - "35": 1901729792.0, - "40": 1901729792.0, - "45": 1901729792.0, - "50": 1901729792.0, - "55": 1901729792.0, - "60": 1901729792.0, - "65": 1901729792.0, - "70": 1901729792.0, - "75": 1901729792.0, - "80": 1901729792.0, - "85": 1901729792.0, - "90": 1901729792.0, - "95": 1901729792.0, - "100": 1901729792.0 + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 14.4466, - "5": 0.23058, - "10": 0.23299, - "15": 0.23075, - "20": 0.23163, - "25": 0.23303, - "30": 0.22991, - "35": 0.22977, - "40": 0.22747, - "45": 0.22974, - "50": 0.52211, - "55": 0.23173, - "60": 0.23326, - "65": 0.23498, - "70": 0.23013, - "75": 0.22876, - "80": 0.22947, - "85": 0.22706, - "90": 0.22612, - "95": 0.22523, - "100": 0.22404 + "1": 7.63807, + "2": 0.24295, + "3": 0.21281, + "4": 0.20931, + "5": 0.20554, + "6": 0.20827, + "7": 0.20618, + "8": 0.20701, + "9": 0.2077, + "10": 0.20875, + "11": 0.20704, + "12": 0.20735, + "13": 0.20734, + "14": 0.20659, + "15": 0.2071, + "16": 0.20766, + "17": 0.20579, + "18": 0.20511, + "19": 0.20563, + "20": 0.20589, + "21": 0.205, + "22": 0.20541, + "23": 0.2062, + "24": 0.20485, + "25": 0.20487, + "26": 0.20458, + "27": 0.20496, + "28": 0.20545, + "29": 0.20623, + "30": 0.20511, + "31": 0.20822, + "32": 0.20482, + "33": 0.20538, + "34": 0.20452, + "35": 0.21063, + "36": 0.20987, + "37": 0.20831, + "38": 0.2088, + "39": 0.20816, + "40": 0.20875, + "41": 0.20857, + "42": 0.20959, + "43": 0.20886, + "44": 0.2086, + "45": 0.20776, + "46": 0.20831, + "47": 0.20853, + "48": 0.2086, + "49": 0.20813, + "50": 0.209, + "51": 0.20574, + "52": 0.19892, + "53": 0.19904, + "54": 0.19867, + "55": 0.19897, + "56": 0.20031, + "57": 0.19874, + "58": 0.19971, + "59": 0.2002, + "60": 0.19847, + "61": 0.19948, + "62": 0.20017, + "63": 0.19926, + "64": 0.19923, + "65": 0.19974, + "66": 0.19915, + "67": 0.19992, + "68": 0.19949, + "69": 0.19842, + "70": 0.19824, + "71": 0.2012, + "72": 0.20144, + "73": 0.20339, + "74": 0.19815, + "75": 0.19802, + "76": 0.19898, + "77": 0.20003, + "78": 0.20017, + "79": 0.20157, + "80": 0.20266, + "81": 0.20004, + "82": 0.19937, + "83": 0.2008, + "84": 0.2009, + "85": 0.20194, + "86": 0.2015, + "87": 0.20004, + "88": 0.20091, + "89": 0.19998, + "90": 0.19993, + "91": 0.20008, + "92": 0.19991, + "93": 0.19979, + "94": 0.19939, + "95": 0.20098, + "96": 0.20045, + "97": 0.19917, + "98": 0.20012, + "99": 0.19963, + "100": 0.19848 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..4378b779118 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.19385, + "2": 0.24311, + "3": 0.20769, + "4": 0.20606, + "5": 0.21034, + "6": 0.20875, + "7": 0.20511, + "8": 0.2045, + "9": 0.20566, + "10": 0.20495, + "11": 0.20684, + "12": 0.20533, + "13": 0.20573, + "14": 0.20691, + "15": 0.20612, + "16": 0.20502, + "17": 0.20553, + "18": 0.20502, + "19": 0.20618, + "20": 0.20587, + "21": 0.20529, + "22": 0.20461, + "23": 0.20575, + "24": 0.2056, + "25": 0.20434, + "26": 0.20473, + "27": 0.20499, + "28": 0.20543, + "29": 0.20597, + "30": 0.20594, + "31": 0.20485, + "32": 0.20584, + "33": 0.20569, + "34": 0.20368, + "35": 0.20375, + "36": 0.20415, + "37": 0.20296, + "38": 0.20379, + "39": 0.20482, + "40": 0.20372, + "41": 0.20369, + "42": 0.2037, + "43": 0.20368, + "44": 0.2036, + "45": 0.20913, + "46": 0.20299, + "47": 0.20291, + "48": 0.20437, + "49": 0.2033, + "50": 0.20293, + "51": 0.21225, + "52": 0.20266, + "53": 0.20231, + "54": 0.20341, + "55": 0.20168, + "56": 0.20234, + "57": 0.20234, + "58": 0.20311, + "59": 0.20312, + "60": 0.20314, + "61": 0.20332, + "62": 0.20225, + "63": 0.20251, + "64": 0.20258, + "65": 0.20214, + "66": 0.2022, + "67": 0.20405, + "68": 0.20192, + "69": 0.2033, + "70": 0.20207, + "71": 0.20216, + "72": 0.20251, + "73": 0.20231, + "74": 0.20203, + "75": 0.20107, + "76": 0.20252, + "77": 0.20263, + "78": 0.20334, + "79": 0.20378, + "80": 0.20072, + "81": 0.20029, + "82": 0.19967, + "83": 0.20354, + "84": 0.20035, + "85": 0.201, + "86": 0.20061, + "87": 0.20132, + "88": 0.20043, + "89": 0.20079, + "90": 0.20125, + "91": 0.20103, + "92": 0.19988, + "93": 0.20137, + "94": 0.20191, + "95": 0.20085, + "96": 0.20182, + "97": 0.20169, + "98": 0.20353, + "99": 0.2004, + "100": 0.20044 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json index 053168da21b..97ea213f560 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92795, "10": 10.90786, "15": 10.88314, "20": 10.77629, "25": 10.59141, "30": 10.39192, "35": 10.29686, "40": 10.0964, "45": 9.84464, "50": 9.90918, "55": 9.87762, "60": 9.49121, "65": 8.94236, "70": 9.72266, "75": 9.41912, "80": 9.40077, "85": 9.61207, "90": 9.81015, "95": 9.51716, "100": 9.4015}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1627.0, "5": 2010.0, "10": 1368.0, "15": 1897.0, "20": 1626.0, "25": 1743.0, "30": 1930.0, "35": 1954.0, "40": 2199.0, "45": 2068.0, "50": 2460.0, "55": 2427.0, "60": 2380.0, "65": 2657.0, "70": 3265.0, "75": 2675.0, "80": 3434.0, "85": 3302.0, "90": 3230.0, "95": 3340.0, "100": 3291.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 487096320.0, "5": 487096320.0, "10": 487096320.0, "15": 487096320.0, "20": 487096320.0, "25": 487096320.0, "30": 487096320.0, "35": 487096320.0, "40": 487096320.0, "45": 487096320.0, "50": 487096320.0, "55": 487096320.0, "60": 487096320.0, "65": 487096320.0, "70": 487096320.0, "75": 487096320.0, "80": 487096320.0, "85": 487096320.0, "90": 487096320.0, "95": 487096320.0, "100": 487096320.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1720084480.0, "5": 1900157952.0, "10": 1900157952.0, "15": 1900157952.0, "20": 1900157952.0, "25": 1900157952.0, "30": 1900157952.0, "35": 1900157952.0, "40": 1900157952.0, "45": 1900157952.0, "50": 1900157952.0, "55": 1900157952.0, "60": 1900157952.0, "65": 1900157952.0, "70": 1900157952.0, "75": 1900157952.0, "80": 1900157952.0, "85": 1900157952.0, "90": 1900157952.0, "95": 1900157952.0, "100": 1900157952.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.26256, "5": 0.218, "10": 0.21609, "15": 0.21516, "20": 0.2149, "25": 0.21592, "30": 0.21243, "35": 0.21175, "40": 0.21098, "45": 0.21116, "50": 0.21161, "55": 0.21221, "60": 0.21264, "65": 0.21113, "70": 0.21176, "75": 0.22341, "80": 0.22245, "85": 0.22222, "90": 0.22203, "95": 0.21302, "100": 0.21583}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.25138, + "2": 0.23075, + "3": 0.20054, + "4": 0.20395, + "5": 0.20085, + "6": 0.19693, + "7": 0.1984, + "8": 0.19691, + "9": 0.19734, + "10": 0.19831, + "11": 0.19755, + "12": 0.20036, + "13": 0.19718, + "14": 0.20205, + "15": 0.19931, + "16": 0.1974, + "17": 0.19891, + "18": 0.19725, + "19": 0.19744, + "20": 0.19621, + "21": 0.19556, + "22": 0.1957, + "23": 0.19653, + "24": 0.19561, + "25": 0.19465, + "26": 0.19483, + "27": 0.19566, + "28": 0.19514, + "29": 0.19571, + "30": 0.19512, + "31": 0.19603, + "32": 0.19794, + "33": 0.19597, + "34": 0.20052, + "35": 0.19938, + "36": 0.19968, + "37": 0.19971, + "38": 0.19989, + "39": 0.20233, + "40": 0.20594, + "41": 0.20596, + "42": 0.20875, + "43": 0.20692, + "44": 0.20224, + "45": 0.20492, + "46": 0.20483, + "47": 0.20404, + "48": 0.20062, + "49": 0.1998, + "50": 0.19944, + "51": 0.21056, + "52": 0.20322, + "53": 0.20394, + "54": 0.20267, + "55": 0.20305, + "56": 0.20261, + "57": 0.20266, + "58": 0.2023, + "59": 0.20259, + "60": 0.20297, + "61": 0.20333, + "62": 0.20344, + "63": 0.20255, + "64": 0.20203, + "65": 0.20288, + "66": 0.20295, + "67": 0.20276, + "68": 0.20255, + "69": 0.20306, + "70": 0.20225, + "71": 0.20236, + "72": 0.20262, + "73": 0.2033, + "74": 0.20279, + "75": 0.20276, + "76": 0.20185, + "77": 0.20283, + "78": 0.20284, + "79": 0.2021, + "80": 0.20273, + "81": 0.20261, + "82": 0.20101, + "83": 0.20222, + "84": 0.20269, + "85": 0.20272, + "86": 0.20286, + "87": 0.20079, + "88": 0.20309, + "89": 0.2026, + "90": 0.20209, + "91": 0.20371, + "92": 0.20302, + "93": 0.20226, + "94": 0.20222, + "95": 0.20289, + "96": 0.20273, + "97": 0.20346, + "98": 0.20283, + "99": 0.20241, + "100": 0.20343 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json new file mode 100644 index 00000000000..f6faa6eb158 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90497, + "5": 10.92805, + "6": 10.93671, + "7": 10.90403, + "8": 10.92231, + "9": 10.91252, + "10": 10.90851, + "11": 10.89338, + "12": 10.9208, + "13": 10.91494, + "14": 10.92144, + "15": 10.88435, + "16": 10.87458, + "17": 10.83918, + "18": 10.87307, + "19": 10.85328, + "20": 10.77494, + "21": 10.74759, + "22": 10.63148, + "23": 10.75623, + "24": 10.65568, + "25": 10.59221, + "26": 10.65325, + "27": 10.64881, + "28": 10.59654, + "29": 10.61012, + "30": 10.39283, + "31": 10.15719, + "32": 10.49221, + "33": 10.47943, + "34": 10.24015, + "35": 10.29711, + "36": 10.24562, + "37": 10.35281, + "38": 10.20533, + "39": 10.40423, + "40": 10.09553, + "41": 10.15279, + "42": 10.21879, + "43": 9.85526, + "44": 9.96244, + "45": 9.84617, + "46": 9.83798, + "47": 10.1388, + "48": 9.85687, + "49": 9.53746, + "50": 9.90877, + "51": 9.84971, + "52": 9.74156, + "53": 10.06324, + "54": 9.94584, + "55": 9.87735, + "56": 9.62744, + "57": 9.4726, + "58": 9.82907, + "59": 9.58298, + "60": 9.49182, + "61": 9.6996, + "62": 9.98091, + "63": 9.37212, + "64": 9.77558, + "65": 8.94327, + "66": 9.69991, + "67": 9.3641, + "68": 9.78706, + "69": 9.78396, + "70": 9.72291, + "71": 9.60749, + "72": 9.58417, + "73": 9.4909, + "74": 8.94863, + "75": 9.41807, + "76": 9.08721, + "77": 10.06284, + "78": 9.729, + "79": 9.37087, + "80": 9.40029, + "81": 9.47753, + "82": 9.69123, + "83": 9.30764, + "84": 9.4125, + "85": 9.61132, + "86": 9.07624, + "87": 9.59459, + "88": 9.74769, + "89": 9.60678, + "90": 9.81079, + "91": 9.34443, + "92": 9.36534, + "93": 9.07741, + "94": 8.82974, + "95": 9.51676, + "96": 9.52545, + "97": 9.31031, + "98": 9.67811, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 59.0, + "2": 59.0, + "3": 57.0, + "4": 55.0, + "5": 68.0, + "6": 60.0, + "7": 58.0, + "8": 63.0, + "9": 62.0, + "10": 47.0, + "11": 57.0, + "12": 48.0, + "13": 68.0, + "14": 58.0, + "15": 75.0, + "16": 67.0, + "17": 59.0, + "18": 56.0, + "19": 72.0, + "20": 52.0, + "21": 60.0, + "22": 69.0, + "23": 77.0, + "24": 58.0, + "25": 71.0, + "26": 68.0, + "27": 55.0, + "28": 94.0, + "29": 57.0, + "30": 66.0, + "31": 60.0, + "32": 64.0, + "33": 69.0, + "34": 73.0, + "35": 73.0, + "36": 72.0, + "37": 89.0, + "38": 72.0, + "39": 75.0, + "40": 82.0, + "41": 82.0, + "42": 74.0, + "43": 90.0, + "44": 69.0, + "45": 83.0, + "46": 68.0, + "47": 74.0, + "48": 95.0, + "49": 81.0, + "50": 83.0, + "51": 77.0, + "52": 100.0, + "53": 71.0, + "54": 67.0, + "55": 70.0, + "56": 83.0, + "57": 74.0, + "58": 106.0, + "59": 72.0, + "60": 98.0, + "61": 67.0, + "62": 73.0, + "63": 77.0, + "64": 94.0, + "65": 82.0, + "66": 87.0, + "67": 65.0, + "68": 78.0, + "69": 59.0, + "70": 102.0, + "71": 82.0, + "72": 60.0, + "73": 96.0, + "74": 61.0, + "75": 64.0, + "76": 70.0, + "77": 84.0, + "78": 93.0, + "79": 102.0, + "80": 71.0, + "81": 88.0, + "82": 85.0, + "83": 75.0, + "84": 69.0, + "85": 84.0, + "86": 66.0, + "87": 93.0, + "88": 96.0, + "89": 73.0, + "90": 77.0, + "91": 66.0, + "92": 86.0, + "93": 63.0, + "94": 60.0, + "95": 70.0, + "96": 65.0, + "97": 67.0, + "98": 96.0, + "99": 54.0, + "100": 77.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 516063744.0, + "2": 516063744.0, + "3": 516063744.0, + "4": 516063744.0, + "5": 516063744.0, + "6": 516063744.0, + "7": 516063744.0, + "8": 516063744.0, + "9": 516063744.0, + "10": 516063744.0, + "11": 516063744.0, + "12": 516063744.0, + "13": 516063744.0, + "14": 516063744.0, + "15": 516063744.0, + "16": 516063744.0, + "17": 516063744.0, + "18": 516063744.0, + "19": 516063744.0, + "20": 516063744.0, + "21": 516063744.0, + "22": 516063744.0, + "23": 516063744.0, + "24": 516063744.0, + "25": 516063744.0, + "26": 516063744.0, + "27": 516063744.0, + "28": 516063744.0, + "29": 516063744.0, + "30": 516063744.0, + "31": 516063744.0, + "32": 516063744.0, + "33": 516063744.0, + "34": 516063744.0, + "35": 516063744.0, + "36": 516063744.0, + "37": 516063744.0, + "38": 516063744.0, + "39": 516063744.0, + "40": 516063744.0, + "41": 516063744.0, + "42": 516063744.0, + "43": 516063744.0, + "44": 516063744.0, + "45": 516063744.0, + "46": 516063744.0, + "47": 516063744.0, + "48": 516063744.0, + "49": 516063744.0, + "50": 516063744.0, + "51": 516063744.0, + "52": 516063744.0, + "53": 516063744.0, + "54": 516063744.0, + "55": 516063744.0, + "56": 516063744.0, + "57": 516063744.0, + "58": 516063744.0, + "59": 516063744.0, + "60": 516063744.0, + "61": 516063744.0, + "62": 516063744.0, + "63": 516063744.0, + "64": 516063744.0, + "65": 516063744.0, + "66": 516063744.0, + "67": 516063744.0, + "68": 516063744.0, + "69": 516063744.0, + "70": 516063744.0, + "71": 516063744.0, + "72": 516063744.0, + "73": 516063744.0, + "74": 516063744.0, + "75": 516063744.0, + "76": 516063744.0, + "77": 516063744.0, + "78": 516063744.0, + "79": 516063744.0, + "80": 516063744.0, + "81": 516063744.0, + "82": 516063744.0, + "83": 516063744.0, + "84": 516063744.0, + "85": 516063744.0, + "86": 516063744.0, + "87": 516063744.0, + "88": 516063744.0, + "89": 516063744.0, + "90": 516063744.0, + "91": 516063744.0, + "92": 516063744.0, + "93": 516063744.0, + "94": 516063744.0, + "95": 516063744.0, + "96": 516063744.0, + "97": 516063744.0, + "98": 516063744.0, + "99": 516063744.0, + "100": 516063744.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1697019392.0, + "2": 1877092864.0, + "3": 1877092864.0, + "4": 1877092864.0, + "5": 1877092864.0, + "6": 1877092864.0, + "7": 1877092864.0, + "8": 1877092864.0, + "9": 1877092864.0, + "10": 1877092864.0, + "11": 1877092864.0, + "12": 1877092864.0, + "13": 1877092864.0, + "14": 1877092864.0, + "15": 1877092864.0, + "16": 1877092864.0, + "17": 1877092864.0, + "18": 1877092864.0, + "19": 1877092864.0, + "20": 1877092864.0, + "21": 1877092864.0, + "22": 1877092864.0, + "23": 1877092864.0, + "24": 1877092864.0, + "25": 1877092864.0, + "26": 1877092864.0, + "27": 1877092864.0, + "28": 1877092864.0, + "29": 1877092864.0, + "30": 1877092864.0, + "31": 1877092864.0, + "32": 1877092864.0, + "33": 1877092864.0, + "34": 1877092864.0, + "35": 1877092864.0, + "36": 1877092864.0, + "37": 1877092864.0, + "38": 1877092864.0, + "39": 1877092864.0, + "40": 1877092864.0, + "41": 1877092864.0, + "42": 1877092864.0, + "43": 1877092864.0, + "44": 1877092864.0, + "45": 1877092864.0, + "46": 1877092864.0, + "47": 1877092864.0, + "48": 1877092864.0, + "49": 1877092864.0, + "50": 1877092864.0, + "51": 1877092864.0, + "52": 1877092864.0, + "53": 1877092864.0, + "54": 1877092864.0, + "55": 1877092864.0, + "56": 1877092864.0, + "57": 1877092864.0, + "58": 1877092864.0, + "59": 1877092864.0, + "60": 1877092864.0, + "61": 1877092864.0, + "62": 1877092864.0, + "63": 1877092864.0, + "64": 1877092864.0, + "65": 1877092864.0, + "66": 1877092864.0, + "67": 1877092864.0, + "68": 1877092864.0, + "69": 1877092864.0, + "70": 1877092864.0, + "71": 1877092864.0, + "72": 1877092864.0, + "73": 1877092864.0, + "74": 1877092864.0, + "75": 1877092864.0, + "76": 1877092864.0, + "77": 1877092864.0, + "78": 1877092864.0, + "79": 1877092864.0, + "80": 1877092864.0, + "81": 1877092864.0, + "82": 1877092864.0, + "83": 1877092864.0, + "84": 1877092864.0, + "85": 1877092864.0, + "86": 1877092864.0, + "87": 1877092864.0, + "88": 1877092864.0, + "89": 1877092864.0, + "90": 1877092864.0, + "91": 1877092864.0, + "92": 1877092864.0, + "93": 1877092864.0, + "94": 1877092864.0, + "95": 1877092864.0, + "96": 1877092864.0, + "97": 1877092864.0, + "98": 1877092864.0, + "99": 1877092864.0, + "100": 1877092864.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.79688, + "2": 0.23426, + "3": 0.19644, + "4": 0.195, + "5": 0.19474, + "6": 0.19503, + "7": 0.19553, + "8": 0.19608, + "9": 0.1942, + "10": 0.19493, + "11": 0.19477, + "12": 0.19436, + "13": 0.19434, + "14": 0.19471, + "15": 0.19477, + "16": 0.19465, + "17": 0.19394, + "18": 0.19358, + "19": 0.19755, + "20": 0.19602, + "21": 0.19501, + "22": 0.1946, + "23": 0.19563, + "24": 0.19522, + "25": 0.19502, + "26": 0.19664, + "27": 0.19509, + "28": 0.19415, + "29": 0.19426, + "30": 0.19405, + "31": 0.19405, + "32": 0.19558, + "33": 0.19428, + "34": 0.19418, + "35": 0.19493, + "36": 0.19478, + "37": 0.19511, + "38": 0.19481, + "39": 0.19506, + "40": 0.19573, + "41": 0.19576, + "42": 0.19504, + "43": 0.19379, + "44": 0.19366, + "45": 0.19309, + "46": 0.19331, + "47": 0.19424, + "48": 0.21152, + "49": 0.2113, + "50": 0.21175, + "51": 0.20739, + "52": 0.19757, + "53": 0.19742, + "54": 0.19771, + "55": 0.19922, + "56": 0.19903, + "57": 0.19826, + "58": 0.19889, + "59": 0.1988, + "60": 0.19894, + "61": 0.19841, + "62": 0.19853, + "63": 0.19879, + "64": 0.19776, + "65": 0.19833, + "66": 0.19911, + "67": 0.19919, + "68": 0.19768, + "69": 0.19855, + "70": 0.19789, + "71": 0.19851, + "72": 0.19815, + "73": 0.19885, + "74": 0.19845, + "75": 0.19827, + "76": 0.19878, + "77": 0.19962, + "78": 0.19912, + "79": 0.19891, + "80": 0.1985, + "81": 0.19933, + "82": 0.1977, + "83": 0.19774, + "84": 0.19798, + "85": 0.19781, + "86": 0.19763, + "87": 0.1977, + "88": 0.19934, + "89": 0.19771, + "90": 0.19835, + "91": 0.19886, + "92": 0.19768, + "93": 0.19818, + "94": 0.19769, + "95": 0.19836, + "96": 0.19833, + "97": 0.19773, + "98": 0.1988, + "99": 0.1973, + "100": 0.19801 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json index 59a50d2f033..96cf765384a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92799, "10": 10.90789, "15": 10.88313, "20": 10.77626, "25": 10.59138, "30": 10.39195, "35": 10.29687, "40": 10.0964, "45": 9.84466, "50": 9.9092, "55": 9.87764, "60": 9.49121, "65": 8.94232, "70": 9.72263, "75": 9.41912, "80": 9.40079, "85": 9.61209, "90": 9.81012, "95": 9.51718, "100": 9.40153}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 68.0, "5": 64.0, "10": 61.0, "15": 58.0, "20": 64.0, "25": 58.0, "30": 85.0, "35": 66.0, "40": 85.0, "45": 84.0, "50": 85.0, "55": 69.0, "60": 67.0, "65": 65.0, "70": 92.0, "75": 51.0, "80": 98.0, "85": 86.0, "90": 83.0, "95": 72.0, "100": 73.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 488144896.0, "5": 488144896.0, "10": 488144896.0, "15": 488144896.0, "20": 488144896.0, "25": 489193472.0, "30": 488144896.0, "35": 488144896.0, "40": 488144896.0, "45": 488144896.0, "50": 488144896.0, "55": 488144896.0, "60": 488144896.0, "65": 488144896.0, "70": 489193472.0, "75": 489193472.0, "80": 489193472.0, "85": 488144896.0, "90": 488144896.0, "95": 488144896.0, "100": 489193472.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2158389248.0, "5": 2340559872.0, "10": 2340559872.0, "15": 2340559872.0, "20": 2340559872.0, "25": 2340559872.0, "30": 2340559872.0, "35": 2340559872.0, "40": 2340559872.0, "45": 2340559872.0, "50": 2340559872.0, "55": 2340559872.0, "60": 2340559872.0, "65": 2340559872.0, "70": 2340559872.0, "75": 2340559872.0, "80": 2340559872.0, "85": 2340559872.0, "90": 2340559872.0, "95": 2340559872.0, "100": 2340559872.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 15.21811, "5": 0.22614, "10": 0.22685, "15": 0.22633, "20": 0.22582, "25": 0.22604, "30": 0.22747, "35": 0.22529, "40": 0.22441, "45": 0.22458, "50": 0.22397, "55": 0.21965, "60": 0.22066, "65": 0.22265, "70": 0.22234, "75": 0.2195, "80": 0.21889, "85": 0.21987, "90": 0.21875, "95": 0.21918, "100": 0.21913}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90497, + "5": 10.92805, + "6": 10.93671, + "7": 10.90403, + "8": 10.92231, + "9": 10.91252, + "10": 10.90851, + "11": 10.89338, + "12": 10.9208, + "13": 10.91494, + "14": 10.92144, + "15": 10.88435, + "16": 10.87458, + "17": 10.83918, + "18": 10.87307, + "19": 10.85328, + "20": 10.77494, + "21": 10.74759, + "22": 10.63148, + "23": 10.75623, + "24": 10.65568, + "25": 10.59221, + "26": 10.65325, + "27": 10.64881, + "28": 10.59654, + "29": 10.61012, + "30": 10.39283, + "31": 10.15719, + "32": 10.49221, + "33": 10.47943, + "34": 10.24015, + "35": 10.29711, + "36": 10.24562, + "37": 10.35281, + "38": 10.20533, + "39": 10.40423, + "40": 10.09553, + "41": 10.15279, + "42": 10.21879, + "43": 9.85526, + "44": 9.96244, + "45": 9.84617, + "46": 9.83798, + "47": 10.1388, + "48": 9.85687, + "49": 9.53746, + "50": 9.90877, + "51": 9.84971, + "52": 9.74156, + "53": 10.06324, + "54": 9.94584, + "55": 9.87735, + "56": 9.62744, + "57": 9.4726, + "58": 9.82907, + "59": 9.58298, + "60": 9.49182, + "61": 9.6996, + "62": 9.98091, + "63": 9.37212, + "64": 9.77558, + "65": 8.94327, + "66": 9.69991, + "67": 9.3641, + "68": 9.78706, + "69": 9.78396, + "70": 9.72291, + "71": 9.60749, + "72": 9.58417, + "73": 9.4909, + "74": 8.94863, + "75": 9.41807, + "76": 9.08721, + "77": 10.06284, + "78": 9.729, + "79": 9.37087, + "80": 9.40029, + "81": 9.47753, + "82": 9.69123, + "83": 9.30764, + "84": 9.4125, + "85": 9.61132, + "86": 9.07624, + "87": 9.59459, + "88": 9.74769, + "89": 9.60678, + "90": 9.81079, + "91": 9.34443, + "92": 9.36534, + "93": 9.07741, + "94": 8.82974, + "95": 9.51676, + "96": 9.52545, + "97": 9.31031, + "98": 9.67811, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 59.0, + "2": 59.0, + "3": 57.0, + "4": 55.0, + "5": 68.0, + "6": 60.0, + "7": 58.0, + "8": 63.0, + "9": 62.0, + "10": 47.0, + "11": 57.0, + "12": 48.0, + "13": 68.0, + "14": 58.0, + "15": 75.0, + "16": 67.0, + "17": 59.0, + "18": 56.0, + "19": 72.0, + "20": 52.0, + "21": 60.0, + "22": 69.0, + "23": 77.0, + "24": 58.0, + "25": 71.0, + "26": 68.0, + "27": 55.0, + "28": 94.0, + "29": 57.0, + "30": 66.0, + "31": 60.0, + "32": 64.0, + "33": 69.0, + "34": 73.0, + "35": 73.0, + "36": 72.0, + "37": 89.0, + "38": 72.0, + "39": 75.0, + "40": 82.0, + "41": 82.0, + "42": 74.0, + "43": 90.0, + "44": 69.0, + "45": 83.0, + "46": 68.0, + "47": 74.0, + "48": 95.0, + "49": 81.0, + "50": 83.0, + "51": 77.0, + "52": 100.0, + "53": 71.0, + "54": 67.0, + "55": 70.0, + "56": 83.0, + "57": 74.0, + "58": 106.0, + "59": 72.0, + "60": 98.0, + "61": 67.0, + "62": 73.0, + "63": 77.0, + "64": 94.0, + "65": 82.0, + "66": 87.0, + "67": 65.0, + "68": 78.0, + "69": 59.0, + "70": 102.0, + "71": 82.0, + "72": 60.0, + "73": 96.0, + "74": 61.0, + "75": 64.0, + "76": 70.0, + "77": 84.0, + "78": 93.0, + "79": 102.0, + "80": 71.0, + "81": 88.0, + "82": 85.0, + "83": 75.0, + "84": 69.0, + "85": 84.0, + "86": 66.0, + "87": 93.0, + "88": 96.0, + "89": 73.0, + "90": 77.0, + "91": 66.0, + "92": 86.0, + "93": 63.0, + "94": 60.0, + "95": 70.0, + "96": 65.0, + "97": 67.0, + "98": 96.0, + "99": 54.0, + "100": 77.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 516063744.0, + "2": 516063744.0, + "3": 516063744.0, + "4": 516063744.0, + "5": 516063744.0, + "6": 516063744.0, + "7": 516063744.0, + "8": 516063744.0, + "9": 516063744.0, + "10": 516063744.0, + "11": 516063744.0, + "12": 516063744.0, + "13": 516063744.0, + "14": 516063744.0, + "15": 516063744.0, + "16": 516063744.0, + "17": 516063744.0, + "18": 516063744.0, + "19": 516063744.0, + "20": 516063744.0, + "21": 516063744.0, + "22": 516063744.0, + "23": 516063744.0, + "24": 516063744.0, + "25": 516063744.0, + "26": 516063744.0, + "27": 516063744.0, + "28": 516063744.0, + "29": 516063744.0, + "30": 516063744.0, + "31": 516063744.0, + "32": 516063744.0, + "33": 516063744.0, + "34": 516063744.0, + "35": 516063744.0, + "36": 516063744.0, + "37": 516063744.0, + "38": 516063744.0, + "39": 516063744.0, + "40": 516063744.0, + "41": 516063744.0, + "42": 516063744.0, + "43": 516063744.0, + "44": 516063744.0, + "45": 516063744.0, + "46": 516063744.0, + "47": 516063744.0, + "48": 516063744.0, + "49": 516063744.0, + "50": 516063744.0, + "51": 516063744.0, + "52": 516063744.0, + "53": 516063744.0, + "54": 516063744.0, + "55": 516063744.0, + "56": 516063744.0, + "57": 516063744.0, + "58": 516063744.0, + "59": 516063744.0, + "60": 516063744.0, + "61": 516063744.0, + "62": 516063744.0, + "63": 516063744.0, + "64": 516063744.0, + "65": 516063744.0, + "66": 516063744.0, + "67": 516063744.0, + "68": 516063744.0, + "69": 516063744.0, + "70": 516063744.0, + "71": 516063744.0, + "72": 516063744.0, + "73": 516063744.0, + "74": 516063744.0, + "75": 516063744.0, + "76": 516063744.0, + "77": 516063744.0, + "78": 516063744.0, + "79": 516063744.0, + "80": 516063744.0, + "81": 516063744.0, + "82": 516063744.0, + "83": 516063744.0, + "84": 516063744.0, + "85": 516063744.0, + "86": 516063744.0, + "87": 516063744.0, + "88": 516063744.0, + "89": 516063744.0, + "90": 516063744.0, + "91": 516063744.0, + "92": 516063744.0, + "93": 516063744.0, + "94": 516063744.0, + "95": 516063744.0, + "96": 516063744.0, + "97": 516063744.0, + "98": 516063744.0, + "99": 516063744.0, + "100": 516063744.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1697019392.0, + "2": 1877092864.0, + "3": 1877092864.0, + "4": 1877092864.0, + "5": 1877092864.0, + "6": 1877092864.0, + "7": 1877092864.0, + "8": 1877092864.0, + "9": 1877092864.0, + "10": 1877092864.0, + "11": 1877092864.0, + "12": 1877092864.0, + "13": 1877092864.0, + "14": 1877092864.0, + "15": 1877092864.0, + "16": 1877092864.0, + "17": 1877092864.0, + "18": 1877092864.0, + "19": 1877092864.0, + "20": 1877092864.0, + "21": 1877092864.0, + "22": 1877092864.0, + "23": 1877092864.0, + "24": 1877092864.0, + "25": 1877092864.0, + "26": 1877092864.0, + "27": 1877092864.0, + "28": 1877092864.0, + "29": 1877092864.0, + "30": 1877092864.0, + "31": 1877092864.0, + "32": 1877092864.0, + "33": 1877092864.0, + "34": 1877092864.0, + "35": 1877092864.0, + "36": 1877092864.0, + "37": 1877092864.0, + "38": 1877092864.0, + "39": 1877092864.0, + "40": 1877092864.0, + "41": 1877092864.0, + "42": 1877092864.0, + "43": 1877092864.0, + "44": 1877092864.0, + "45": 1877092864.0, + "46": 1877092864.0, + "47": 1877092864.0, + "48": 1877092864.0, + "49": 1877092864.0, + "50": 1877092864.0, + "51": 1877092864.0, + "52": 1877092864.0, + "53": 1877092864.0, + "54": 1877092864.0, + "55": 1877092864.0, + "56": 1877092864.0, + "57": 1877092864.0, + "58": 1877092864.0, + "59": 1877092864.0, + "60": 1877092864.0, + "61": 1877092864.0, + "62": 1877092864.0, + "63": 1877092864.0, + "64": 1877092864.0, + "65": 1877092864.0, + "66": 1877092864.0, + "67": 1877092864.0, + "68": 1877092864.0, + "69": 1877092864.0, + "70": 1877092864.0, + "71": 1877092864.0, + "72": 1877092864.0, + "73": 1877092864.0, + "74": 1877092864.0, + "75": 1877092864.0, + "76": 1877092864.0, + "77": 1877092864.0, + "78": 1877092864.0, + "79": 1877092864.0, + "80": 1877092864.0, + "81": 1877092864.0, + "82": 1877092864.0, + "83": 1877092864.0, + "84": 1877092864.0, + "85": 1877092864.0, + "86": 1877092864.0, + "87": 1877092864.0, + "88": 1877092864.0, + "89": 1877092864.0, + "90": 1877092864.0, + "91": 1877092864.0, + "92": 1877092864.0, + "93": 1877092864.0, + "94": 1877092864.0, + "95": 1877092864.0, + "96": 1877092864.0, + "97": 1877092864.0, + "98": 1877092864.0, + "99": 1877092864.0, + "100": 1877092864.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.10882, + "2": 0.24563, + "3": 0.21507, + "4": 0.21225, + "5": 0.21165, + "6": 0.21127, + "7": 0.21406, + "8": 0.21402, + "9": 0.21175, + "10": 0.19518, + "11": 0.19565, + "12": 0.19461, + "13": 0.19428, + "14": 0.19385, + "15": 0.19329, + "16": 0.19311, + "17": 0.19391, + "18": 0.19383, + "19": 0.19364, + "20": 0.19408, + "21": 0.19327, + "22": 0.19729, + "23": 0.19599, + "24": 0.19601, + "25": 0.1965, + "26": 0.19683, + "27": 0.19626, + "28": 0.19667, + "29": 0.1989, + "30": 0.19644, + "31": 0.19728, + "32": 0.19614, + "33": 0.1973, + "34": 0.1971, + "35": 0.19674, + "36": 0.19628, + "37": 0.19578, + "38": 0.19629, + "39": 0.19673, + "40": 0.19712, + "41": 0.19593, + "42": 0.1969, + "43": 0.19639, + "44": 0.20378, + "45": 0.19737, + "46": 0.19738, + "47": 0.19532, + "48": 0.19579, + "49": 0.19617, + "50": 0.19695, + "51": 0.20318, + "52": 0.19428, + "53": 0.19415, + "54": 0.19663, + "55": 0.19266, + "56": 0.19426, + "57": 0.19455, + "58": 0.19473, + "59": 0.19413, + "60": 0.19467, + "61": 0.19511, + "62": 0.19475, + "63": 0.19464, + "64": 0.19452, + "65": 0.19445, + "66": 0.19395, + "67": 0.19423, + "68": 0.19431, + "69": 0.19512, + "70": 0.1941, + "71": 0.19453, + "72": 0.19467, + "73": 0.19615, + "74": 0.19355, + "75": 0.19419, + "76": 0.19407, + "77": 0.19455, + "78": 0.19511, + "79": 0.19498, + "80": 0.19577, + "81": 0.19399, + "82": 0.19362, + "83": 0.19425, + "84": 0.19418, + "85": 0.19432, + "86": 0.20057, + "87": 0.19522, + "88": 0.19447, + "89": 0.19472, + "90": 0.19377, + "91": 0.19433, + "92": 0.19432, + "93": 0.19456, + "94": 0.19394, + "95": 0.19417, + "96": 0.19476, + "97": 0.19423, + "98": 0.19401, + "99": 0.19403, + "100": 0.19364 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json new file mode 100644 index 00000000000..6a8c9ac9b1d --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 482499072.0, + "2": 482499072.0, + "3": 482499072.0, + "4": 482499072.0, + "5": 482499072.0, + "6": 482499072.0, + "7": 482499072.0, + "8": 482499072.0, + "9": 482499072.0, + "10": 482499072.0, + "11": 482499072.0, + "12": 482499072.0, + "13": 482499072.0, + "14": 482499072.0, + "15": 482499072.0, + "16": 482499072.0, + "17": 482499072.0, + "18": 482499072.0, + "19": 482499072.0, + "20": 482499072.0, + "21": 482499072.0, + "22": 482499072.0, + "23": 482499072.0, + "24": 482499072.0, + "25": 482499072.0, + "26": 482499072.0, + "27": 482499072.0, + "28": 482499072.0, + "29": 482499072.0, + "30": 482499072.0, + "31": 482499072.0, + "32": 482499072.0, + "33": 482499072.0, + "34": 482499072.0, + "35": 482499072.0, + "36": 482499072.0, + "37": 482499072.0, + "38": 482499072.0, + "39": 482499072.0, + "40": 482499072.0, + "41": 482499072.0, + "42": 482499072.0, + "43": 482499072.0, + "44": 482499072.0, + "45": 482499072.0, + "46": 482499072.0, + "47": 482499072.0, + "48": 482499072.0, + "49": 482499072.0, + "50": 482499072.0, + "51": 482499072.0, + "52": 482499072.0, + "53": 482499072.0, + "54": 482499072.0, + "55": 482499072.0, + "56": 482499072.0, + "57": 482499072.0, + "58": 482499072.0, + "59": 482499072.0, + "60": 482499072.0, + "61": 482499072.0, + "62": 482499072.0, + "63": 482499072.0, + "64": 482499072.0, + "65": 482499072.0, + "66": 482499072.0, + "67": 482499072.0, + "68": 482499072.0, + "69": 482499072.0, + "70": 482499072.0, + "71": 482499072.0, + "72": 482499072.0, + "73": 482499072.0, + "74": 482499072.0, + "75": 482499072.0, + "76": 482499072.0, + "77": 482499072.0, + "78": 482499072.0, + "79": 482499072.0, + "80": 482499072.0, + "81": 482499072.0, + "82": 482499072.0, + "83": 482499072.0, + "84": 482499072.0, + "85": 482499072.0, + "86": 482499072.0, + "87": 482499072.0, + "88": 482499072.0, + "89": 482499072.0, + "90": 482499072.0, + "91": 482499072.0, + "92": 482499072.0, + "93": 482499072.0, + "94": 482499072.0, + "95": 482499072.0, + "96": 482499072.0, + "97": 482499072.0, + "98": 482499072.0, + "99": 482499072.0, + "100": 482499072.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1222004224.0, + "2": 1401029120.0, + "3": 1401029120.0, + "4": 1401029120.0, + "5": 1401029120.0, + "6": 1401029120.0, + "7": 1401029120.0, + "8": 1401029120.0, + "9": 1401029120.0, + "10": 1401029120.0, + "11": 1401029120.0, + "12": 1401029120.0, + "13": 1401029120.0, + "14": 1401029120.0, + "15": 1401029120.0, + "16": 1401029120.0, + "17": 1401029120.0, + "18": 1401029120.0, + "19": 1401029120.0, + "20": 1401029120.0, + "21": 1401029120.0, + "22": 1401029120.0, + "23": 1401029120.0, + "24": 1401029120.0, + "25": 1401029120.0, + "26": 1401029120.0, + "27": 1401029120.0, + "28": 1401029120.0, + "29": 1401029120.0, + "30": 1401029120.0, + "31": 1401029120.0, + "32": 1401029120.0, + "33": 1401029120.0, + "34": 1401029120.0, + "35": 1401029120.0, + "36": 1401029120.0, + "37": 1401029120.0, + "38": 1401029120.0, + "39": 1401029120.0, + "40": 1401029120.0, + "41": 1401029120.0, + "42": 1401029120.0, + "43": 1401029120.0, + "44": 1401029120.0, + "45": 1401029120.0, + "46": 1401029120.0, + "47": 1401029120.0, + "48": 1401029120.0, + "49": 1401029120.0, + "50": 1401029120.0, + "51": 1401029120.0, + "52": 1401029120.0, + "53": 1401029120.0, + "54": 1401029120.0, + "55": 1401029120.0, + "56": 1401029120.0, + "57": 1401029120.0, + "58": 1401029120.0, + "59": 1401029120.0, + "60": 1401029120.0, + "61": 1401029120.0, + "62": 1401029120.0, + "63": 1401029120.0, + "64": 1401029120.0, + "65": 1401029120.0, + "66": 1401029120.0, + "67": 1401029120.0, + "68": 1401029120.0, + "69": 1401029120.0, + "70": 1401029120.0, + "71": 1401029120.0, + "72": 1401029120.0, + "73": 1401029120.0, + "74": 1401029120.0, + "75": 1401029120.0, + "76": 1401029120.0, + "77": 1401029120.0, + "78": 1401029120.0, + "79": 1401029120.0, + "80": 1401029120.0, + "81": 1401029120.0, + "82": 1401029120.0, + "83": 1401029120.0, + "84": 1401029120.0, + "85": 1401029120.0, + "86": 1401029120.0, + "87": 1401029120.0, + "88": 1401029120.0, + "89": 1401029120.0, + "90": 1401029120.0, + "91": 1401029120.0, + "92": 1401029120.0, + "93": 1401029120.0, + "94": 1401029120.0, + "95": 1401029120.0, + "96": 1401029120.0, + "97": 1401029120.0, + "98": 1401029120.0, + "99": 1401029120.0, + "100": 1401029120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.51486, + "2": 0.24464, + "3": 0.21825, + "4": 0.21579, + "5": 0.21661, + "6": 0.22001, + "7": 0.22285, + "8": 0.21811, + "9": 0.21904, + "10": 0.22002, + "11": 0.21823, + "12": 0.21823, + "13": 0.21801, + "14": 0.2183, + "15": 0.21803, + "16": 0.21896, + "17": 0.22491, + "18": 0.21905, + "19": 0.21958, + "20": 0.21884, + "21": 0.21909, + "22": 0.21942, + "23": 0.21797, + "24": 0.21855, + "25": 0.21901, + "26": 0.21791, + "27": 0.2177, + "28": 0.2177, + "29": 0.21821, + "30": 0.21868, + "31": 0.21834, + "32": 0.22204, + "33": 0.22091, + "34": 0.21934, + "35": 0.21783, + "36": 0.21796, + "37": 0.21775, + "38": 0.22135, + "39": 0.22172, + "40": 0.21756, + "41": 0.2177, + "42": 0.21859, + "43": 0.21823, + "44": 0.21755, + "45": 0.21785, + "46": 0.21785, + "47": 0.21753, + "48": 0.21811, + "49": 0.21778, + "50": 0.218, + "51": 0.22344, + "52": 0.21422, + "53": 0.21383, + "54": 0.21399, + "55": 0.21397, + "56": 0.21365, + "57": 0.21387, + "58": 0.20817, + "59": 0.20847, + "60": 0.20906, + "61": 0.20895, + "62": 0.20893, + "63": 0.20812, + "64": 0.20751, + "65": 0.20797, + "66": 0.2078, + "67": 0.20815, + "68": 0.20705, + "69": 0.20775, + "70": 0.20774, + "71": 0.2069, + "72": 0.20722, + "73": 0.20718, + "74": 0.2072, + "75": 0.20697, + "76": 0.20716, + "77": 0.20758, + "78": 0.20779, + "79": 0.20721, + "80": 0.20748, + "81": 0.20797, + "82": 0.20844, + "83": 0.20732, + "84": 0.20744, + "85": 0.20706, + "86": 0.20715, + "87": 0.20731, + "88": 0.20708, + "89": 0.20778, + "90": 0.20769, + "91": 0.2074, + "92": 0.20797, + "93": 0.20707, + "94": 0.2072, + "95": 0.20676, + "96": 0.20732, + "97": 0.20728, + "98": 0.20675, + "99": 0.2068, + "100": 0.20724 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json index 339f9b5b061..d6134cdcc5a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.92705, "5": 10.92795, "10": 10.90786, "15": 10.88314, "20": 10.77629, "25": 10.59141, "30": 10.39192, "35": 10.29686, "40": 10.0964, "45": 9.84464, "50": 9.90918, "55": 9.87762, "60": 9.49121, "65": 8.94236, "70": 9.72266, "75": 9.41912, "80": 9.40077, "85": 9.61207, "90": 9.81015, "95": 9.51716, "100": 9.4015}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1627.0, "5": 2010.0, "10": 1368.0, "15": 1897.0, "20": 1626.0, "25": 1743.0, "30": 1930.0, "35": 1954.0, "40": 2199.0, "45": 2068.0, "50": 2460.0, "55": 2427.0, "60": 2380.0, "65": 2657.0, "70": 3265.0, "75": 2675.0, "80": 3434.0, "85": 3302.0, "90": 3230.0, "95": 3340.0, "100": 3291.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 486047744.0, "5": 486047744.0, "10": 486047744.0, "15": 486047744.0, "20": 486047744.0, "25": 486047744.0, "30": 486047744.0, "35": 486047744.0, "40": 486047744.0, "45": 486047744.0, "50": 486047744.0, "55": 486047744.0, "60": 486047744.0, "65": 486047744.0, "70": 486047744.0, "75": 486047744.0, "80": 486047744.0, "85": 486047744.0, "90": 486047744.0, "95": 486047744.0, "100": 486047744.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1715890176.0, "5": 1895963648.0, "10": 1895963648.0, "15": 1895963648.0, "20": 1895963648.0, "25": 1895963648.0, "30": 1895963648.0, "35": 1895963648.0, "40": 1895963648.0, "45": 1895963648.0, "50": 1895963648.0, "55": 1895963648.0, "60": 1895963648.0, "65": 1895963648.0, "70": 1895963648.0, "75": 1895963648.0, "80": 1895963648.0, "85": 1895963648.0, "90": 1895963648.0, "95": 1895963648.0, "100": 1895963648.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.92657, "5": 0.21658, "10": 0.20915, "15": 0.2079, "20": 0.21595, "25": 0.21334, "30": 0.21734, "35": 0.21646, "40": 0.21045, "45": 0.20732, "50": 0.2058, "55": 0.20444, "60": 0.20486, "65": 0.20503, "70": 0.20714, "75": 0.20517, "80": 0.2067, "85": 0.20664, "90": 0.20761, "95": 0.20379, "100": 0.20501}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 482499072.0, + "2": 482499072.0, + "3": 482499072.0, + "4": 482499072.0, + "5": 482499072.0, + "6": 482499072.0, + "7": 482499072.0, + "8": 482499072.0, + "9": 482499072.0, + "10": 482499072.0, + "11": 482499072.0, + "12": 482499072.0, + "13": 482499072.0, + "14": 482499072.0, + "15": 482499072.0, + "16": 482499072.0, + "17": 482499072.0, + "18": 482499072.0, + "19": 482499072.0, + "20": 482499072.0, + "21": 482499072.0, + "22": 482499072.0, + "23": 482499072.0, + "24": 482499072.0, + "25": 482499072.0, + "26": 482499072.0, + "27": 482499072.0, + "28": 482499072.0, + "29": 482499072.0, + "30": 482499072.0, + "31": 482499072.0, + "32": 482499072.0, + "33": 482499072.0, + "34": 482499072.0, + "35": 482499072.0, + "36": 482499072.0, + "37": 482499072.0, + "38": 482499072.0, + "39": 482499072.0, + "40": 482499072.0, + "41": 482499072.0, + "42": 482499072.0, + "43": 482499072.0, + "44": 482499072.0, + "45": 482499072.0, + "46": 482499072.0, + "47": 482499072.0, + "48": 482499072.0, + "49": 482499072.0, + "50": 482499072.0, + "51": 482499072.0, + "52": 482499072.0, + "53": 482499072.0, + "54": 482499072.0, + "55": 482499072.0, + "56": 482499072.0, + "57": 482499072.0, + "58": 482499072.0, + "59": 482499072.0, + "60": 482499072.0, + "61": 482499072.0, + "62": 482499072.0, + "63": 482499072.0, + "64": 482499072.0, + "65": 482499072.0, + "66": 482499072.0, + "67": 482499072.0, + "68": 482499072.0, + "69": 482499072.0, + "70": 482499072.0, + "71": 482499072.0, + "72": 482499072.0, + "73": 482499072.0, + "74": 482499072.0, + "75": 482499072.0, + "76": 482499072.0, + "77": 482499072.0, + "78": 482499072.0, + "79": 482499072.0, + "80": 482499072.0, + "81": 482499072.0, + "82": 482499072.0, + "83": 482499072.0, + "84": 482499072.0, + "85": 482499072.0, + "86": 482499072.0, + "87": 482499072.0, + "88": 482499072.0, + "89": 482499072.0, + "90": 482499072.0, + "91": 482499072.0, + "92": 482499072.0, + "93": 482499072.0, + "94": 482499072.0, + "95": 482499072.0, + "96": 482499072.0, + "97": 482499072.0, + "98": 482499072.0, + "99": 482499072.0, + "100": 482499072.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1222004224.0, + "2": 1401029120.0, + "3": 1401029120.0, + "4": 1401029120.0, + "5": 1401029120.0, + "6": 1401029120.0, + "7": 1401029120.0, + "8": 1401029120.0, + "9": 1401029120.0, + "10": 1401029120.0, + "11": 1401029120.0, + "12": 1401029120.0, + "13": 1401029120.0, + "14": 1401029120.0, + "15": 1401029120.0, + "16": 1401029120.0, + "17": 1401029120.0, + "18": 1401029120.0, + "19": 1401029120.0, + "20": 1401029120.0, + "21": 1401029120.0, + "22": 1401029120.0, + "23": 1401029120.0, + "24": 1401029120.0, + "25": 1401029120.0, + "26": 1401029120.0, + "27": 1401029120.0, + "28": 1401029120.0, + "29": 1401029120.0, + "30": 1401029120.0, + "31": 1401029120.0, + "32": 1401029120.0, + "33": 1401029120.0, + "34": 1401029120.0, + "35": 1401029120.0, + "36": 1401029120.0, + "37": 1401029120.0, + "38": 1401029120.0, + "39": 1401029120.0, + "40": 1401029120.0, + "41": 1401029120.0, + "42": 1401029120.0, + "43": 1401029120.0, + "44": 1401029120.0, + "45": 1401029120.0, + "46": 1401029120.0, + "47": 1401029120.0, + "48": 1401029120.0, + "49": 1401029120.0, + "50": 1401029120.0, + "51": 1401029120.0, + "52": 1401029120.0, + "53": 1401029120.0, + "54": 1401029120.0, + "55": 1401029120.0, + "56": 1401029120.0, + "57": 1401029120.0, + "58": 1401029120.0, + "59": 1401029120.0, + "60": 1401029120.0, + "61": 1401029120.0, + "62": 1401029120.0, + "63": 1401029120.0, + "64": 1401029120.0, + "65": 1401029120.0, + "66": 1401029120.0, + "67": 1401029120.0, + "68": 1401029120.0, + "69": 1401029120.0, + "70": 1401029120.0, + "71": 1401029120.0, + "72": 1401029120.0, + "73": 1401029120.0, + "74": 1401029120.0, + "75": 1401029120.0, + "76": 1401029120.0, + "77": 1401029120.0, + "78": 1401029120.0, + "79": 1401029120.0, + "80": 1401029120.0, + "81": 1401029120.0, + "82": 1401029120.0, + "83": 1401029120.0, + "84": 1401029120.0, + "85": 1401029120.0, + "86": 1401029120.0, + "87": 1401029120.0, + "88": 1401029120.0, + "89": 1401029120.0, + "90": 1401029120.0, + "91": 1401029120.0, + "92": 1401029120.0, + "93": 1401029120.0, + "94": 1401029120.0, + "95": 1401029120.0, + "96": 1401029120.0, + "97": 1401029120.0, + "98": 1401029120.0, + "99": 1401029120.0, + "100": 1401029120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.20377, + "2": 0.2288, + "3": 0.19616, + "4": 0.19587, + "5": 0.19737, + "6": 0.19775, + "7": 0.19658, + "8": 0.19621, + "9": 0.19557, + "10": 0.19534, + "11": 0.19453, + "12": 0.1949, + "13": 0.19522, + "14": 0.19865, + "15": 0.20415, + "16": 0.19686, + "17": 0.1985, + "18": 0.19858, + "19": 0.19709, + "20": 0.19609, + "21": 0.19758, + "22": 0.19837, + "23": 0.19786, + "24": 0.19688, + "25": 0.1972, + "26": 0.19859, + "27": 0.19814, + "28": 0.1989, + "29": 0.1984, + "30": 0.19783, + "31": 0.19727, + "32": 0.19754, + "33": 0.19648, + "34": 0.19977, + "35": 0.19847, + "36": 0.19696, + "37": 0.20498, + "38": 0.20415, + "39": 0.20225, + "40": 0.19712, + "41": 0.19751, + "42": 0.19764, + "43": 0.19738, + "44": 0.19703, + "45": 0.19703, + "46": 0.19814, + "47": 0.19757, + "48": 0.19759, + "49": 0.19688, + "50": 0.20181, + "51": 0.22215, + "52": 0.2134, + "53": 0.2129, + "54": 0.2133, + "55": 0.21255, + "56": 0.21221, + "57": 0.21233, + "58": 0.2124, + "59": 0.21242, + "60": 0.21258, + "61": 0.21219, + "62": 0.21255, + "63": 0.21385, + "64": 0.2127, + "65": 0.21252, + "66": 0.21191, + "67": 0.21327, + "68": 0.21176, + "69": 0.2127, + "70": 0.21284, + "71": 0.21291, + "72": 0.21265, + "73": 0.21221, + "74": 0.21387, + "75": 0.21247, + "76": 0.21204, + "77": 0.21169, + "78": 0.21259, + "79": 0.21196, + "80": 0.21204, + "81": 0.21211, + "82": 0.21314, + "83": 0.21268, + "84": 0.21291, + "85": 0.21328, + "86": 0.2128, + "87": 0.21213, + "88": 0.21192, + "89": 0.21242, + "90": 0.21253, + "91": 0.21252, + "92": 0.21236, + "93": 0.21254, + "94": 0.21255, + "95": 0.21209, + "96": 0.21345, + "97": 0.21202, + "98": 0.21234, + "99": 0.21237, + "100": 0.21317 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..b276f849ced --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.85961, + "2": 10.85774, + "3": 10.85857, + "4": 10.84879, + "5": 10.88247, + "6": 10.88701, + "7": 10.86152, + "8": 10.86792, + "9": 10.86512, + "10": 10.83742, + "11": 10.87758, + "12": 10.87232, + "13": 10.87849, + "14": 10.88848, + "15": 10.82866, + "16": 10.8307, + "17": 10.79817, + "18": 10.82417, + "19": 10.81181, + "20": 10.72691, + "21": 10.7183, + "22": 10.57059, + "23": 10.72817, + "24": 10.61403, + "25": 10.55842, + "26": 10.60676, + "27": 10.61661, + "28": 10.57482, + "29": 10.59587, + "30": 10.37815, + "31": 10.13027, + "32": 10.47778, + "33": 10.46854, + "34": 10.22658, + "35": 10.27984, + "36": 10.22842, + "37": 10.35309, + "38": 10.20319, + "39": 10.4096, + "40": 10.0888, + "41": 10.15947, + "42": 10.2147, + "43": 9.84971, + "44": 9.96501, + "45": 9.82545, + "46": 9.83484, + "47": 10.15515, + "48": 9.84462, + "49": 9.52863, + "50": 9.91253 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1816.0, + "2": 1884.0, + "3": 1906.0, + "4": 1901.0, + "5": 2061.0, + "6": 2186.0, + "7": 2145.0, + "8": 1775.0, + "9": 2106.0, + "10": 1630.0, + "11": 2236.0, + "12": 2015.0, + "13": 2168.0, + "14": 2021.0, + "15": 2088.0, + "16": 2152.0, + "17": 1976.0, + "18": 1952.0, + "19": 2046.0, + "20": 1740.0, + "21": 2073.0, + "22": 1810.0, + "23": 2186.0, + "24": 1839.0, + "25": 1681.0, + "26": 1962.0, + "27": 2009.0, + "28": 2204.0, + "29": 2264.0, + "30": 2117.0, + "31": 1873.0, + "32": 2178.0, + "33": 2372.0, + "34": 2127.0, + "35": 2225.0, + "36": 2140.0, + "37": 2649.0, + "38": 2602.0, + "39": 2607.0, + "40": 2574.0, + "41": 2503.0, + "42": 2532.0, + "43": 2285.0, + "44": 2313.0, + "45": 2423.0, + "46": 2563.0, + "47": 2856.0, + "48": 2629.0, + "49": 2566.0, + "50": 2651.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1449634304.0, + "2": 1516021248.0, + "3": 1516021248.0, + "4": 1516021248.0, + "5": 1516021248.0, + "6": 1516021248.0, + "7": 1516021248.0, + "8": 1516021248.0, + "9": 1516021248.0, + "10": 1516021248.0, + "11": 1516021248.0, + "12": 1516021248.0, + "13": 1516021248.0, + "14": 1516021248.0, + "15": 1516021248.0, + "16": 1516021248.0, + "17": 1516021248.0, + "18": 1516021248.0, + "19": 1516021248.0, + "20": 1516021248.0, + "21": 1516021248.0, + "22": 1516021248.0, + "23": 1516021248.0, + "24": 1516021248.0, + "25": 1516021248.0, + "26": 1516021248.0, + "27": 1516021248.0, + "28": 1516021248.0, + "29": 1516021248.0, + "30": 1516021248.0, + "31": 1516021248.0, + "32": 1516021248.0, + "33": 1516021248.0, + "34": 1516021248.0, + "35": 1516021248.0, + "36": 1516021248.0, + "37": 1516021248.0, + "38": 1516021248.0, + "39": 1516021248.0, + "40": 1516021248.0, + "41": 1516021248.0, + "42": 1516021248.0, + "43": 1516021248.0, + "44": 1516021248.0, + "45": 1516021248.0, + "46": 1516021248.0, + "47": 1516021248.0, + "48": 1516021248.0, + "49": 1516021248.0, + "50": 1516021248.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.56269, + "2": 0.39732, + "3": 0.33618, + "4": 0.33333, + "5": 0.33363, + "6": 0.33477, + "7": 0.33191, + "8": 0.33146, + "9": 0.33302, + "10": 0.33134, + "11": 0.33151, + "12": 0.33246, + "13": 0.33088, + "14": 0.3303, + "15": 0.33382, + "16": 0.33069, + "17": 0.33094, + "18": 0.33118, + "19": 0.33644, + "20": 0.3305, + "21": 0.33088, + "22": 0.33128, + "23": 0.33088, + "24": 0.33099, + "25": 0.33033, + "26": 0.33052, + "27": 0.33152, + "28": 0.33036, + "29": 0.33036, + "30": 0.33116, + "31": 0.33249, + "32": 0.33284, + "33": 0.33031, + "34": 0.33138, + "35": 0.33068, + "36": 0.33706, + "37": 0.33218, + "38": 0.33033, + "39": 0.33382, + "40": 0.34395, + "41": 0.33213, + "42": 0.3316, + "43": 0.33261, + "44": 0.33133, + "45": 0.33097, + "46": 0.33137, + "47": 0.33245, + "48": 0.33171, + "49": 0.33138, + "50": 0.33185 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index 28f4a9f3d58..5dd18b2b701 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.86122, "5": 10.88242, "10": 10.83506, "15": 10.82738, "20": 10.72743, "25": 10.55753, "30": 10.37895, "35": 10.28321, "40": 10.08785, "45": 9.82625, "50": 9.91327}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1778.0, "5": 2219.0, "10": 1530.0, "15": 2125.0, "20": 1867.0, "25": 1773.0, "30": 2129.0, "35": 2169.0, "40": 2486.0, "45": 2335.0, "50": 2687.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 269891584.0, "5": 269891584.0, "10": 269891584.0, "15": 269891584.0, "20": 269891584.0, "25": 269891584.0, "30": 269891584.0, "35": 269891584.0, "40": 269891584.0, "45": 269891584.0, "50": 269891584.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1450730496.0, "5": 1515676160.0, "10": 1515676672.0, "15": 1515676672.0, "20": 1515676672.0, "25": 1515676672.0, "30": 1515676672.0, "35": 1515676672.0, "40": 1515676672.0, "45": 1515676672.0, "50": 1515676672.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 11.10997, "5": 0.31331, "10": 0.32223, "15": 0.31308, "20": 0.3166, "25": 0.31574, "30": 0.31074, "35": 0.31512, "40": 0.31604, "45": 0.32221, "50": 0.31241}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.85961, + "2": 10.85774, + "3": 10.85857, + "4": 10.84879, + "5": 10.88247, + "6": 10.88701, + "7": 10.86152, + "8": 10.86792, + "9": 10.86512, + "10": 10.83742, + "11": 10.87758, + "12": 10.87232, + "13": 10.87849, + "14": 10.88848, + "15": 10.82866, + "16": 10.8307, + "17": 10.79817, + "18": 10.82417, + "19": 10.81181, + "20": 10.72691, + "21": 10.7183, + "22": 10.57059, + "23": 10.72817, + "24": 10.61403, + "25": 10.55842, + "26": 10.60676, + "27": 10.61661, + "28": 10.57482, + "29": 10.59587, + "30": 10.37815, + "31": 10.13027, + "32": 10.47778, + "33": 10.46854, + "34": 10.22658, + "35": 10.27984, + "36": 10.22842, + "37": 10.35309, + "38": 10.20319, + "39": 10.4096, + "40": 10.0888, + "41": 10.15947, + "42": 10.2147, + "43": 9.84971, + "44": 9.96501, + "45": 9.82545, + "46": 9.83484, + "47": 10.15515, + "48": 9.84462, + "49": 9.52863, + "50": 9.91253 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1816.0, + "2": 1884.0, + "3": 1906.0, + "4": 1901.0, + "5": 2061.0, + "6": 2186.0, + "7": 2145.0, + "8": 1775.0, + "9": 2106.0, + "10": 1630.0, + "11": 2236.0, + "12": 2015.0, + "13": 2168.0, + "14": 2021.0, + "15": 2088.0, + "16": 2152.0, + "17": 1976.0, + "18": 1952.0, + "19": 2046.0, + "20": 1740.0, + "21": 2073.0, + "22": 1810.0, + "23": 2186.0, + "24": 1839.0, + "25": 1681.0, + "26": 1962.0, + "27": 2009.0, + "28": 2204.0, + "29": 2264.0, + "30": 2117.0, + "31": 1873.0, + "32": 2178.0, + "33": 2372.0, + "34": 2127.0, + "35": 2225.0, + "36": 2140.0, + "37": 2649.0, + "38": 2602.0, + "39": 2607.0, + "40": 2574.0, + "41": 2503.0, + "42": 2532.0, + "43": 2285.0, + "44": 2313.0, + "45": 2423.0, + "46": 2563.0, + "47": 2856.0, + "48": 2629.0, + "49": 2566.0, + "50": 2651.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1449634304.0, + "2": 1516021248.0, + "3": 1516021248.0, + "4": 1516021248.0, + "5": 1516021248.0, + "6": 1516021248.0, + "7": 1516021248.0, + "8": 1516021248.0, + "9": 1516021248.0, + "10": 1516021248.0, + "11": 1516021248.0, + "12": 1516021248.0, + "13": 1516021248.0, + "14": 1516021248.0, + "15": 1516021248.0, + "16": 1516021248.0, + "17": 1516021248.0, + "18": 1516021248.0, + "19": 1516021248.0, + "20": 1516021248.0, + "21": 1516021248.0, + "22": 1516021248.0, + "23": 1516021248.0, + "24": 1516021248.0, + "25": 1516021248.0, + "26": 1516021248.0, + "27": 1516021248.0, + "28": 1516021248.0, + "29": 1516021248.0, + "30": 1516021248.0, + "31": 1516021248.0, + "32": 1516021248.0, + "33": 1516021248.0, + "34": 1516021248.0, + "35": 1516021248.0, + "36": 1516021248.0, + "37": 1516021248.0, + "38": 1516021248.0, + "39": 1516021248.0, + "40": 1516021248.0, + "41": 1516021248.0, + "42": 1516021248.0, + "43": 1516021248.0, + "44": 1516021248.0, + "45": 1516021248.0, + "46": 1516021248.0, + "47": 1516021248.0, + "48": 1516021248.0, + "49": 1516021248.0, + "50": 1516021248.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.8193, + "2": 0.36983, + "3": 0.32405, + "4": 0.32179, + "5": 0.32037, + "6": 0.32162, + "7": 0.32479, + "8": 0.32031, + "9": 0.32398, + "10": 0.32296, + "11": 0.32125, + "12": 0.32185, + "13": 0.323, + "14": 0.32307, + "15": 0.32035, + "16": 0.31953, + "17": 0.32119, + "18": 0.32536, + "19": 0.32368, + "20": 0.32071, + "21": 0.32043, + "22": 0.32093, + "23": 0.32096, + "24": 0.31999, + "25": 0.32046, + "26": 0.31988, + "27": 0.32184, + "28": 0.32107, + "29": 0.32078, + "30": 0.32174, + "31": 0.32345, + "32": 0.32975, + "33": 0.32181, + "34": 0.32294, + "35": 0.32426, + "36": 0.32184, + "37": 0.32175, + "38": 0.32222, + "39": 0.32058, + "40": 0.32111, + "41": 0.33546, + "42": 0.32505, + "43": 0.32502, + "44": 0.32486, + "45": 0.32683, + "46": 0.32331, + "47": 0.322, + "48": 0.32205, + "49": 0.32128, + "50": 0.32053 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..e2187c31d21 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971620864.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.72292, + "2": 0.40079, + "3": 0.30233, + "4": 0.29437, + "5": 0.29362, + "6": 0.29166, + "7": 0.29282, + "8": 0.29142, + "9": 0.29651, + "10": 0.29169, + "11": 0.29163, + "12": 0.29014, + "13": 0.29007, + "14": 0.28821, + "15": 0.28623, + "16": 0.28688, + "17": 0.28583, + "18": 0.2859, + "19": 0.28593, + "20": 0.28532, + "21": 0.28573, + "22": 0.28605, + "23": 0.2847, + "24": 0.28621, + "25": 0.28664, + "26": 0.28556, + "27": 0.2872, + "28": 0.28573, + "29": 0.28695, + "30": 0.28636, + "31": 0.28686, + "32": 0.28621, + "33": 0.28593, + "34": 0.28506, + "35": 0.28606, + "36": 0.28626, + "37": 0.28499, + "38": 0.28674, + "39": 0.28971, + "40": 0.28955, + "41": 0.28907, + "42": 0.29041, + "43": 0.28912, + "44": 0.28905, + "45": 0.28882, + "46": 0.28956, + "47": 0.28907, + "48": 0.28954, + "49": 0.28925, + "50": 0.28916, + "51": 0.2978, + "52": 0.29215, + "53": 0.30386, + "54": 0.28798, + "55": 0.30697, + "56": 0.32648, + "57": 0.29076, + "58": 0.29115, + "59": 0.29247, + "60": 0.29092, + "61": 0.29317, + "62": 0.28506, + "63": 0.28662, + "64": 0.2855, + "65": 0.28592, + "66": 0.28679, + "67": 0.28596, + "68": 0.28605, + "69": 0.28703, + "70": 0.28617, + "71": 0.28619, + "72": 0.28686, + "73": 0.28763, + "74": 0.28583, + "75": 0.28621, + "76": 0.28619, + "77": 0.28487, + "78": 0.28523, + "79": 0.28634, + "80": 0.28584, + "81": 0.28564, + "82": 0.28528, + "83": 0.28536, + "84": 0.28516, + "85": 0.28552, + "86": 0.28537, + "87": 0.2846, + "88": 0.28517, + "89": 0.28625, + "90": 0.28499, + "91": 0.28525, + "92": 0.28549, + "93": 0.28511, + "94": 0.28501, + "95": 0.28642, + "96": 0.28514, + "97": 0.28481, + "98": 0.2852, + "99": 0.28627, + "100": 0.2861 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json index 82a2437ad9d..0cc3719ac53 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86122, "5": 10.88242, "10": 10.83506, "15": 10.82738, "20": 10.72743, "25": 10.55753, "30": 10.37895, "35": 10.28321, "40": 10.08785, "45": 9.82625, "50": 9.91327, "55": 9.87788, "60": 9.50874, "65": 8.95103, "70": 9.73167, "75": 9.43681, "80": 9.41156, "85": 9.61613, "90": 9.8217, "95": 9.5191, "100": 9.40588}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1778.0, "5": 2219.0, "10": 1530.0, "15": 2125.0, "20": 1867.0, "25": 1773.0, "30": 2129.0, "35": 2169.0, "40": 2486.0, "45": 2335.0, "50": 2687.0, "55": 2652.0, "60": 2765.0, "65": 2946.0, "70": 3629.0, "75": 2702.0, "80": 3866.0, "85": 3517.0, "90": 3349.0, "95": 3530.0, "100": 3530.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 269891584.0, "5": 269891584.0, "10": 269891584.0, "15": 269891584.0, "20": 269891584.0, "25": 269891584.0, "30": 269891584.0, "35": 269891584.0, "40": 269891584.0, "45": 269891584.0, "50": 269891584.0, "55": 269891584.0, "60": 269891584.0, "65": 269891584.0, "70": 269891584.0, "75": 269891584.0, "80": 269891584.0, "85": 269891584.0, "90": 269891584.0, "95": 269891584.0, "100": 269891584.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1448634368.0, "5": 1515676672.0, "10": 1515676672.0, "15": 1515676672.0, "20": 1515676672.0, "25": 1515676672.0, "30": 1515676672.0, "35": 1515676672.0, "40": 1515676672.0, "45": 1515676672.0, "50": 1515676672.0, "55": 1515676672.0, "60": 1515676672.0, "65": 1515676672.0, "70": 1515676672.0, "75": 1515676672.0, "80": 1515676672.0, "85": 1515676672.0, "90": 1515676672.0, "95": 1515676672.0, "100": 1515676672.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.54144, "5": 0.31998, "10": 0.30758, "15": 0.30768, "20": 0.30227, "25": 0.30186, "30": 0.3093, "35": 0.3126, "40": 0.31097, "45": 0.30984, "50": 0.3102, "55": 0.30366, "60": 0.30466, "65": 0.3097, "70": 0.30131, "75": 0.30287, "80": 0.30479, "85": 0.30276, "90": 0.3042, "95": 0.30274, "100": 0.30335}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971619840.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.22987, + "2": 0.54363, + "3": 0.2879, + "4": 0.28745, + "5": 0.28509, + "6": 0.28364, + "7": 0.28401, + "8": 0.28235, + "9": 0.28321, + "10": 0.32258, + "11": 0.28697, + "12": 0.27808, + "13": 0.27857, + "14": 0.27833, + "15": 0.28035, + "16": 0.27859, + "17": 0.27841, + "18": 0.27879, + "19": 0.27874, + "20": 0.27806, + "21": 0.27812, + "22": 0.2783, + "23": 0.27919, + "24": 0.27841, + "25": 0.27852, + "26": 0.27871, + "27": 0.27891, + "28": 0.28056, + "29": 0.27909, + "30": 0.2797, + "31": 0.27903, + "32": 0.27895, + "33": 0.27929, + "34": 0.27838, + "35": 0.27904, + "36": 0.2787, + "37": 0.28662, + "38": 0.27812, + "39": 0.27805, + "40": 0.27846, + "41": 0.27884, + "42": 0.27807, + "43": 0.27794, + "44": 0.27825, + "45": 0.28052, + "46": 0.27856, + "47": 0.27832, + "48": 0.27799, + "49": 0.2783, + "50": 0.27861, + "51": 0.2915, + "52": 0.28668, + "53": 0.28545, + "54": 0.28632, + "55": 0.28616, + "56": 0.28735, + "57": 0.28738, + "58": 0.28556, + "59": 0.28453, + "60": 0.28543, + "61": 0.28452, + "62": 0.28404, + "63": 0.28542, + "64": 0.28492, + "65": 0.28488, + "66": 0.2861, + "67": 0.286, + "68": 0.28505, + "69": 0.28531, + "70": 0.28377, + "71": 0.28517, + "72": 0.28454, + "73": 0.2853, + "74": 0.28678, + "75": 0.28484, + "76": 0.28523, + "77": 0.28548, + "78": 0.28488, + "79": 0.28559, + "80": 0.28528, + "81": 0.28479, + "82": 0.28465, + "83": 0.28506, + "84": 0.28493, + "85": 0.28486, + "86": 0.28572, + "87": 0.28404, + "88": 0.28473, + "89": 0.28431, + "90": 0.28945, + "91": 0.28446, + "92": 0.28489, + "93": 0.28474, + "94": 0.28484, + "95": 0.28526, + "96": 0.28573, + "97": 0.28411, + "98": 0.28402, + "99": 0.28413, + "100": 0.28454 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..a39e093badf --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971619840.0, + "2": 1036172288.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.83564, + "2": 0.46115, + "3": 0.29577, + "4": 0.29295, + "5": 0.2937, + "6": 0.29159, + "7": 0.29645, + "8": 0.3016, + "9": 0.29961, + "10": 0.29819, + "11": 0.29865, + "12": 0.2945, + "13": 0.29585, + "14": 0.29535, + "15": 0.29488, + "16": 0.29419, + "17": 0.30001, + "18": 0.29304, + "19": 0.29268, + "20": 0.29243, + "21": 0.29254, + "22": 0.29287, + "23": 0.2907, + "24": 0.29139, + "25": 0.29157, + "26": 0.29169, + "27": 0.29335, + "28": 0.29054, + "29": 0.29193, + "30": 0.29255, + "31": 0.29123, + "32": 0.29014, + "33": 0.29168, + "34": 0.29286, + "35": 0.29078, + "36": 0.29177, + "37": 0.29116, + "38": 0.29148, + "39": 0.29215, + "40": 0.2925, + "41": 0.29112, + "42": 0.29245, + "43": 0.28949, + "44": 0.28901, + "45": 0.28833, + "46": 0.28915, + "47": 0.28932, + "48": 0.28813, + "49": 0.28806, + "50": 0.28818, + "51": 0.30469, + "52": 0.29982, + "53": 0.29764, + "54": 0.29724, + "55": 0.29321, + "56": 0.29333, + "57": 0.29533, + "58": 0.29391, + "59": 0.29338, + "60": 0.29347, + "61": 0.29437, + "62": 0.29327, + "63": 0.29354, + "64": 0.31415, + "65": 0.2977, + "66": 0.2961, + "67": 0.29576, + "68": 0.2944, + "69": 0.29403, + "70": 0.29434, + "71": 0.29505, + "72": 0.2967, + "73": 0.30974, + "74": 0.2985, + "75": 0.29801, + "76": 0.29505, + "77": 0.29554, + "78": 0.29652, + "79": 0.29594, + "80": 0.2958, + "81": 0.29561, + "82": 0.29502, + "83": 0.29476, + "84": 0.29486, + "85": 0.29516, + "86": 0.29553, + "87": 0.29666, + "88": 0.29626, + "89": 0.29583, + "90": 0.29691, + "91": 0.29506, + "92": 0.29865, + "93": 0.29628, + "94": 0.30563, + "95": 0.29582, + "96": 0.29572, + "97": 0.29519, + "98": 0.29708, + "99": 0.29725, + "100": 0.29684 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index 54bb3cbea8d..1641ae309dc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.86122, - "2": 10.85774, - "3": 10.86039, - "4": 10.84813, - "5": 10.88242, - "6": 10.88645, - "7": 10.86227, - "8": 10.86932, - "9": 10.86444, - "10": 10.83506, - "11": 10.87765, - "12": 10.87384, - "13": 10.87945, - "14": 10.88919, - "15": 10.82738, - "16": 10.83105, - "17": 10.79888, - "18": 10.82441, - "19": 10.81363, - "20": 10.72743, - "21": 10.71638, - "22": 10.57153, - "23": 10.7269, - "24": 10.61223, - "25": 10.55753, - "26": 10.60603, - "27": 10.61792, - "28": 10.57695, - "29": 10.59633, - "30": 10.37895, - "31": 10.13125, - "32": 10.47822, - "33": 10.46894, - "34": 10.22715, - "35": 10.28321, - "36": 10.22751, - "37": 10.35397, - "38": 10.20483, - "39": 10.40755, - "40": 10.08785, - "41": 10.1591, - "42": 10.21601, - "43": 9.84821, - "44": 9.9651, - "45": 9.82625, - "46": 9.83468, - "47": 10.15337, - "48": 9.84529, - "49": 9.52926, - "50": 9.91327, - "51": 9.8517, - "52": 9.74686, - "53": 10.07204, - "54": 9.95738, - "55": 9.87788, - "56": 9.62943, - "57": 9.48988, - "58": 9.83265, - "59": 9.58831, - "60": 9.50874, - "61": 9.69495, - "62": 9.99373, - "63": 9.377, - "64": 9.78004, - "65": 8.95103, - "66": 9.71392, + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, "67": 9.37884, - "68": 9.78831, - "69": 9.79096, - "70": 9.73167, - "71": 9.61776, - "72": 9.59099, - "73": 9.49436, - "74": 8.95001, - "75": 9.43681, - "76": 9.09852, - "77": 10.06447, - "78": 9.72944, - "79": 9.37805, - "80": 9.41156, - "81": 9.48537, - "82": 9.69592, - "83": 9.31981, - "84": 9.42306, - "85": 9.61613, - "86": 9.07185, - "87": 9.59282, - "88": 9.75055, - "89": 9.61194, - "90": 9.8217, - "91": 9.35308, - "92": 9.36305, - "93": 9.08788, - "94": 8.83439, - "95": 9.5191, - "96": 9.52647, - "97": 9.31412, - "98": 9.67541, - "99": 8.88941, - "100": 9.40588 + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1778.0, - "2": 1875.0, - "3": 1879.0, - "4": 1912.0, - "5": 2219.0, - "6": 2163.0, - "7": 2113.0, - "8": 1747.0, - "9": 2049.0, - "10": 1530.0, - "11": 2113.0, - "12": 1959.0, - "13": 2134.0, - "14": 2055.0, - "15": 2125.0, - "16": 2139.0, - "17": 1988.0, - "18": 1892.0, - "19": 1991.0, - "20": 1867.0, - "21": 2023.0, - "22": 1865.0, - "23": 2185.0, - "24": 1774.0, - "25": 1773.0, - "26": 1990.0, - "27": 2061.0, - "28": 2215.0, - "29": 2186.0, - "30": 2129.0, - "31": 1794.0, - "32": 2109.0, - "33": 2422.0, - "34": 2135.0, - "35": 2169.0, - "36": 2127.0, - "37": 2432.0, - "38": 2490.0, - "39": 2495.0, - "40": 2486.0, - "41": 2465.0, - "42": 2535.0, - "43": 2216.0, - "44": 2407.0, - "45": 2335.0, - "46": 2617.0, - "47": 2830.0, - "48": 2480.0, - "49": 2492.0, - "50": 2687.0, - "51": 2863.0, - "52": 2881.0, - "53": 3220.0, - "54": 2894.0, - "55": 2652.0, - "56": 3006.0, - "57": 2561.0, - "58": 3273.0, - "59": 3039.0, - "60": 2765.0, - "61": 3310.0, - "62": 2936.0, - "63": 2630.0, - "64": 3230.0, - "65": 2946.0, - "66": 3500.0, - "67": 2976.0, - "68": 2944.0, - "69": 3117.0, - "70": 3629.0, - "71": 3255.0, - "72": 2633.0, - "73": 3338.0, - "74": 2172.0, - "75": 2702.0, - "76": 3162.0, - "77": 3850.0, - "78": 3590.0, - "79": 3658.0, - "80": 3866.0, - "81": 3976.0, - "82": 3680.0, - "83": 3153.0, - "84": 3586.0, - "85": 3517.0, - "86": 3137.0, - "87": 4177.0, - "88": 3589.0, - "89": 3849.0, - "90": 3349.0, - "91": 2936.0, - "92": 3526.0, - "93": 2965.0, - "94": 3772.0, - "95": 3530.0, - "96": 3774.0, - "97": 3636.0, - "98": 4064.0, - "99": 3394.0, - "100": 3530.0 + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 269891584.0, - "2": 269891584.0, - "3": 269891584.0, - "4": 269891584.0, - "5": 269891584.0, - "6": 269891584.0, - "7": 269891584.0, - "8": 269891584.0, - "9": 269891584.0, - "10": 269891584.0, - "11": 269891584.0, - "12": 269891584.0, - "13": 269891584.0, - "14": 269891584.0, - "15": 269891584.0, - "16": 269891584.0, - "17": 269891584.0, - "18": 269891584.0, - "19": 269891584.0, - "20": 269891584.0, - "21": 269891584.0, - "22": 269891584.0, - "23": 269891584.0, - "24": 269891584.0, - "25": 269891584.0, - "26": 269891584.0, - "27": 269891584.0, - "28": 269891584.0, - "29": 269891584.0, - "30": 269891584.0, - "31": 269891584.0, - "32": 269891584.0, - "33": 269891584.0, - "34": 269891584.0, - "35": 269891584.0, - "36": 269891584.0, - "37": 269891584.0, - "38": 269891584.0, - "39": 269891584.0, - "40": 269891584.0, - "41": 269891584.0, - "42": 269891584.0, - "43": 269891584.0, - "44": 269891584.0, - "45": 269891584.0, - "46": 269891584.0, - "47": 269891584.0, - "48": 269891584.0, - "49": 269891584.0, - "50": 269891584.0, - "51": 269891584.0, - "52": 269891584.0, - "53": 269891584.0, - "54": 269891584.0, - "55": 269891584.0, - "56": 269891584.0, - "57": 269891584.0, - "58": 269891584.0, - "59": 269891584.0, - "60": 269891584.0, - "61": 269891584.0, - "62": 269891584.0, - "63": 269891584.0, - "64": 269891584.0, - "65": 269891584.0, - "66": 269891584.0, - "67": 269891584.0, - "68": 269891584.0, - "69": 269891584.0, - "70": 269891584.0, - "71": 269891584.0, - "72": 269891584.0, - "73": 269891584.0, - "74": 269891584.0, - "75": 269891584.0, - "76": 269891584.0, - "77": 269891584.0, - "78": 269891584.0, - "79": 269891584.0, - "80": 269891584.0, - "81": 269891584.0, - "82": 269891584.0, - "83": 269891584.0, - "84": 269891584.0, - "85": 269891584.0, - "86": 269891584.0, - "87": 269891584.0, - "88": 269891584.0, - "89": 269891584.0, - "90": 269891584.0, - "91": 269891584.0, - "92": 269891584.0, - "93": 269891584.0, - "94": 269891584.0, - "95": 269891584.0, - "96": 269891584.0, - "97": 269891584.0, - "98": 269891584.0, - "99": 269891584.0, - "100": 269891584.0 + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1448634368.0, - "2": 1515676672.0, - "3": 1515676672.0, - "4": 1515676672.0, - "5": 1515676672.0, - "6": 1515676672.0, - "7": 1515676672.0, - "8": 1515676672.0, - "9": 1515676672.0, - "10": 1515676672.0, - "11": 1515676672.0, - "12": 1515676672.0, - "13": 1515676672.0, - "14": 1515676672.0, - "15": 1515676672.0, - "16": 1515676672.0, - "17": 1515676672.0, - "18": 1515676672.0, - "19": 1515676672.0, - "20": 1515676672.0, - "21": 1515676672.0, - "22": 1515676672.0, - "23": 1515676672.0, - "24": 1515676672.0, - "25": 1515676672.0, - "26": 1515676672.0, - "27": 1515676672.0, - "28": 1515676672.0, - "29": 1515676672.0, - "30": 1515676672.0, - "31": 1515676672.0, - "32": 1515676672.0, - "33": 1515676672.0, - "34": 1515676672.0, - "35": 1515676672.0, - "36": 1515676672.0, - "37": 1515676672.0, - "38": 1515676672.0, - "39": 1515676672.0, - "40": 1515676672.0, - "41": 1515676672.0, - "42": 1515676672.0, - "43": 1515676672.0, - "44": 1515676672.0, - "45": 1515676672.0, - "46": 1515676672.0, - "47": 1515676672.0, - "48": 1515676672.0, - "49": 1515676672.0, - "50": 1515676672.0, - "51": 1515676672.0, - "52": 1515676672.0, - "53": 1515676672.0, - "54": 1515676672.0, - "55": 1515676672.0, - "56": 1515676672.0, - "57": 1515676672.0, - "58": 1515676672.0, - "59": 1515676672.0, - "60": 1515676672.0, - "61": 1515676672.0, - "62": 1515676672.0, - "63": 1515676672.0, - "64": 1515676672.0, - "65": 1515676672.0, - "66": 1515676672.0, - "67": 1515676672.0, - "68": 1515676672.0, - "69": 1515676672.0, - "70": 1515676672.0, - "71": 1515676672.0, - "72": 1515676672.0, - "73": 1515676672.0, - "74": 1515676672.0, - "75": 1515676672.0, - "76": 1515676672.0, - "77": 1515676672.0, - "78": 1515676672.0, - "79": 1515676672.0, - "80": 1515676672.0, - "81": 1515676672.0, - "82": 1515676672.0, - "83": 1515676672.0, - "84": 1515676672.0, - "85": 1515676672.0, - "86": 1515676672.0, - "87": 1515676672.0, - "88": 1515676672.0, - "89": 1515676672.0, - "90": 1515676672.0, - "91": 1515676672.0, - "92": 1515676672.0, - "93": 1515676672.0, - "94": 1515676672.0, - "95": 1515676672.0, - "96": 1515676672.0, - "97": 1515676672.0, - "98": 1515676672.0, - "99": 1515676672.0, - "100": 1515676672.0 + "1": 971619840.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.29271, - "2": 0.42506, - "3": 0.68343, - "4": 0.36852, - "5": 0.35945, - "6": 0.70082, - "7": 0.36184, - "8": 0.36666, - "9": 0.36956, - "10": 0.36948, - "11": 0.34035, - "12": 0.33106, - "13": 0.32678, - "14": 0.50153, - "15": 0.32624, - "16": 0.32544, - "17": 0.33191, - "18": 0.32618, - "19": 0.3263, - "20": 0.33069, - "21": 0.32595, - "22": 0.3257, - "23": 0.33264, - "24": 0.32517, - "25": 0.32475, - "26": 0.33346, - "27": 0.33354, - "28": 0.32383, - "29": 0.33025, - "30": 0.32292, - "31": 0.32259, - "32": 0.33133, - "33": 0.32233, - "34": 0.32205, - "35": 0.32577, - "36": 0.33027, - "37": 0.32369, - "38": 0.3231, - "39": 0.32941, - "40": 0.32272, - "41": 0.32419, - "42": 0.32862, - "43": 0.32341, - "44": 0.32437, - "45": 0.3291, - "46": 0.32245, - "47": 0.32412, - "48": 0.32928, - "49": 0.32252, - "50": 0.3232, - "51": 0.3288, - "52": 0.32267, - "53": 0.32323, - "54": 0.33682, - "55": 0.32632, - "56": 0.32697, - "57": 0.33895, - "58": 0.32618, - "59": 0.32589, - "60": 0.3322, - "61": 0.3251, - "62": 0.32521, - "63": 0.33036, - "64": 0.32444, - "65": 0.32508, - "66": 0.33114, - "67": 0.32315, - "68": 0.32508, - "69": 0.3303, - "70": 0.32701, - "71": 0.32493, - "72": 0.32932, - "73": 0.32763, - "74": 0.32474, - "75": 0.32636, - "76": 0.33103, - "77": 0.32433, - "78": 0.32583, - "79": 0.33332, - "80": 0.32445, - "81": 0.32512, - "82": 0.33846, - "83": 0.32647, - "84": 0.32584, - "85": 0.33063, - "86": 0.32531, - "87": 0.32597, - "88": 0.33536, - "89": 0.32529, - "90": 0.32619, - "91": 0.33191, - "92": 0.32549, - "93": 0.32565, - "94": 0.33549, - "95": 0.32239, - "96": 0.32249, - "97": 0.32967, - "98": 0.3225, - "99": 0.32206, - "100": 0.32856 + "1": 5.18846, + "2": 0.36168, + "3": 0.29466, + "4": 0.29234, + "5": 0.29276, + "6": 0.29792, + "7": 0.29352, + "8": 0.2936, + "9": 0.29237, + "10": 0.29769, + "11": 0.29346, + "12": 0.29527, + "13": 0.29315, + "14": 0.29363, + "15": 0.29305, + "16": 0.29641, + "17": 0.29489, + "18": 0.29861, + "19": 0.29574, + "20": 0.29312, + "21": 0.29388, + "22": 0.29283, + "23": 0.29431, + "24": 0.29335, + "25": 0.29314, + "26": 0.29296, + "27": 0.29356, + "28": 0.29335, + "29": 0.29568, + "30": 0.29411, + "31": 0.29379, + "32": 0.29273, + "33": 0.29354, + "34": 0.29433, + "35": 0.29411, + "36": 0.29363, + "37": 0.2938, + "38": 0.29351, + "39": 0.29356, + "40": 0.29298, + "41": 0.29347, + "42": 0.29413, + "43": 0.29252, + "44": 0.29273, + "45": 0.29334, + "46": 0.29356, + "47": 0.29382, + "48": 0.29398, + "49": 0.2936, + "50": 0.29316, + "51": 0.29514, + "52": 0.28916, + "53": 0.29005, + "54": 0.28929, + "55": 0.28956, + "56": 0.28848, + "57": 0.28858, + "58": 0.28768, + "59": 0.28853, + "60": 0.29008, + "61": 0.2889, + "62": 0.28847, + "63": 0.28786, + "64": 0.28795, + "65": 0.28879, + "66": 0.28923, + "67": 0.28915, + "68": 0.28861, + "69": 0.28895, + "70": 0.28885, + "71": 0.28882, + "72": 0.28775, + "73": 0.28792, + "74": 0.28799, + "75": 0.28754, + "76": 0.28789, + "77": 0.2888, + "78": 0.28929, + "79": 0.28854, + "80": 0.28894, + "81": 0.28751, + "82": 0.28815, + "83": 0.2885, + "84": 0.28813, + "85": 0.28933, + "86": 0.28794, + "87": 0.28758, + "88": 0.28772, + "89": 0.28903, + "90": 0.28798, + "91": 0.28695, + "92": 0.28757, + "93": 0.28831, + "94": 0.28828, + "95": 0.28871, + "96": 0.28746, + "97": 0.28767, + "98": 0.28881, + "99": 0.2875, + "100": 0.28775 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json new file mode 100644 index 00000000000..d8c6189b1ed --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85966, + "2": 10.85779, + "3": 10.85806, + "4": 10.85111, + "5": 10.88314, + "6": 10.8873, + "7": 10.86205, + "8": 10.86979, + "9": 10.86489, + "10": 10.83833, + "11": 10.87881, + "12": 10.87319, + "13": 10.87924, + "14": 10.88927, + "15": 10.83002, + "16": 10.83231, + "17": 10.80059, + "18": 10.82546, + "19": 10.81132, + "20": 10.73125, + "21": 10.72292, + "22": 10.57559, + "23": 10.73211, + "24": 10.61836, + "25": 10.56602, + "26": 10.61211, + "27": 10.62224, + "28": 10.58011, + "29": 10.59905, + "30": 10.38311, + "31": 10.14181, + "32": 10.48118, + "33": 10.47322, + "34": 10.23556, + "35": 10.28738, + "36": 10.23493, + "37": 10.35785, + "38": 10.20631, + "39": 10.41397, + "40": 10.09217, + "41": 10.16577, + "42": 10.21856, + "43": 9.85974, + "44": 9.97115, + "45": 9.83404, + "46": 9.84483, + "47": 10.16033, + "48": 9.85135, + "49": 9.53781, + "50": 9.91577, + "51": 9.8567, + "52": 9.75178, + "53": 10.07652, + "54": 9.96084, + "55": 9.88221, + "56": 9.63206, + "57": 9.49147, + "58": 9.83408, + "59": 9.59352, + "60": 9.51388, + "61": 9.69802, + "62": 9.99154, + "63": 9.3723, + "64": 9.77839, + "65": 8.95518, + "66": 9.70976, + "67": 9.38198, + "68": 9.78701, + "69": 9.793, + "70": 9.73033, + "71": 9.61752, + "72": 9.58459, + "73": 9.48958, + "74": 8.94015, + "75": 9.43092, + "76": 9.09168, + "77": 10.06222, + "78": 9.72696, + "79": 9.37408, + "80": 9.40676, + "81": 9.47995, + "82": 9.69225, + "83": 9.31299, + "84": 9.41921, + "85": 9.61096, + "86": 9.06853, + "87": 9.59119, + "88": 9.74582, + "89": 9.60624, + "90": 9.81746, + "91": 9.34247, + "92": 9.35856, + "93": 9.07894, + "94": 8.82753, + "95": 9.51606, + "96": 9.52063, + "97": 9.31097, + "98": 9.67055, + "99": 8.88626, + "100": 9.40485 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1756.0, + "2": 2002.0, + "3": 1950.0, + "4": 1894.0, + "5": 2227.0, + "6": 2168.0, + "7": 2021.0, + "8": 1775.0, + "9": 2051.0, + "10": 1611.0, + "11": 2147.0, + "12": 1965.0, + "13": 2185.0, + "14": 2052.0, + "15": 2134.0, + "16": 2077.0, + "17": 2080.0, + "18": 1914.0, + "19": 1993.0, + "20": 1823.0, + "21": 2142.0, + "22": 1847.0, + "23": 2340.0, + "24": 1831.0, + "25": 1790.0, + "26": 1923.0, + "27": 2066.0, + "28": 2268.0, + "29": 2261.0, + "30": 2043.0, + "31": 1759.0, + "32": 2122.0, + "33": 2389.0, + "34": 2186.0, + "35": 2312.0, + "36": 2176.0, + "37": 2626.0, + "38": 2471.0, + "39": 2624.0, + "40": 2556.0, + "41": 2475.0, + "42": 2504.0, + "43": 2326.0, + "44": 2307.0, + "45": 2355.0, + "46": 2625.0, + "47": 2809.0, + "48": 2621.0, + "49": 2670.0, + "50": 2699.0, + "51": 2829.0, + "52": 2809.0, + "53": 3230.0, + "54": 2864.0, + "55": 2706.0, + "56": 2917.0, + "57": 2529.0, + "58": 3339.0, + "59": 3051.0, + "60": 2623.0, + "61": 3287.0, + "62": 2913.0, + "63": 2639.0, + "64": 3154.0, + "65": 2856.0, + "66": 3465.0, + "67": 2934.0, + "68": 2985.0, + "69": 3298.0, + "70": 3653.0, + "71": 3260.0, + "72": 2684.0, + "73": 3232.0, + "74": 2191.0, + "75": 2766.0, + "76": 3335.0, + "77": 3793.0, + "78": 3608.0, + "79": 3384.0, + "80": 3782.0, + "81": 3969.0, + "82": 3640.0, + "83": 3237.0, + "84": 3606.0, + "85": 3553.0, + "86": 3160.0, + "87": 4130.0, + "88": 3430.0, + "89": 3818.0, + "90": 3363.0, + "91": 3041.0, + "92": 3524.0, + "93": 3060.0, + "94": 3575.0, + "95": 3463.0, + "96": 3921.0, + "97": 3597.0, + "98": 4039.0, + "99": 3435.0, + "100": 3548.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 368387584.0, + "2": 368387584.0, + "3": 368387584.0, + "4": 368387584.0, + "5": 368387584.0, + "6": 368387584.0, + "7": 368387584.0, + "8": 368387584.0, + "9": 368387584.0, + "10": 368387584.0, + "11": 368387584.0, + "12": 368387584.0, + "13": 368387584.0, + "14": 368387584.0, + "15": 368387584.0, + "16": 368387584.0, + "17": 368387584.0, + "18": 368387584.0, + "19": 368387584.0, + "20": 368387584.0, + "21": 368387584.0, + "22": 368387584.0, + "23": 368387584.0, + "24": 368387584.0, + "25": 368387584.0, + "26": 368387584.0, + "27": 368387584.0, + "28": 368387584.0, + "29": 368387584.0, + "30": 368387584.0, + "31": 368387584.0, + "32": 368387584.0, + "33": 368387584.0, + "34": 368387584.0, + "35": 368387584.0, + "36": 368387584.0, + "37": 368387584.0, + "38": 368387584.0, + "39": 368387584.0, + "40": 368387584.0, + "41": 368387584.0, + "42": 368387584.0, + "43": 368387584.0, + "44": 368387584.0, + "45": 368387584.0, + "46": 368387584.0, + "47": 368387584.0, + "48": 368387584.0, + "49": 368387584.0, + "50": 368387584.0, + "51": 368387584.0, + "52": 368387584.0, + "53": 368387584.0, + "54": 368387584.0, + "55": 368387584.0, + "56": 368387584.0, + "57": 368387584.0, + "58": 368387584.0, + "59": 368387584.0, + "60": 368387584.0, + "61": 368387584.0, + "62": 368387584.0, + "63": 368387584.0, + "64": 368387584.0, + "65": 368387584.0, + "66": 368387584.0, + "67": 368387584.0, + "68": 368387584.0, + "69": 368387584.0, + "70": 368387584.0, + "71": 368387584.0, + "72": 368387584.0, + "73": 368387584.0, + "74": 368387584.0, + "75": 368387584.0, + "76": 368387584.0, + "77": 368387584.0, + "78": 368387584.0, + "79": 368387584.0, + "80": 368387584.0, + "81": 368387584.0, + "82": 368387584.0, + "83": 368387584.0, + "84": 368387584.0, + "85": 368387584.0, + "86": 368387584.0, + "87": 368387584.0, + "88": 368387584.0, + "89": 368387584.0, + "90": 368387584.0, + "91": 368387584.0, + "92": 368387584.0, + "93": 368387584.0, + "94": 368387584.0, + "95": 368387584.0, + "96": 368387584.0, + "97": 368387584.0, + "98": 368387584.0, + "99": 368387584.0, + "100": 368387584.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1030075904.0, + "2": 1161062912.0, + "3": 1161062912.0, + "4": 1161062912.0, + "5": 1161062912.0, + "6": 1161062912.0, + "7": 1161062912.0, + "8": 1161062912.0, + "9": 1161062912.0, + "10": 1161062912.0, + "11": 1161062912.0, + "12": 1161062912.0, + "13": 1161062912.0, + "14": 1161062912.0, + "15": 1161062912.0, + "16": 1161062912.0, + "17": 1161062912.0, + "18": 1161062912.0, + "19": 1161062912.0, + "20": 1161062912.0, + "21": 1161062912.0, + "22": 1161062912.0, + "23": 1161062912.0, + "24": 1161062912.0, + "25": 1161062912.0, + "26": 1161062912.0, + "27": 1161062912.0, + "28": 1161062912.0, + "29": 1161062912.0, + "30": 1161062912.0, + "31": 1161062912.0, + "32": 1161062912.0, + "33": 1161062912.0, + "34": 1161062912.0, + "35": 1161062912.0, + "36": 1161062912.0, + "37": 1161062912.0, + "38": 1161062912.0, + "39": 1161062912.0, + "40": 1161062912.0, + "41": 1161062912.0, + "42": 1161062912.0, + "43": 1161062912.0, + "44": 1161062912.0, + "45": 1161062912.0, + "46": 1161062912.0, + "47": 1161062912.0, + "48": 1161062912.0, + "49": 1161062912.0, + "50": 1161062912.0, + "51": 1161062912.0, + "52": 1161062912.0, + "53": 1161062912.0, + "54": 1161062912.0, + "55": 1161062912.0, + "56": 1161062912.0, + "57": 1161062912.0, + "58": 1161062912.0, + "59": 1161062912.0, + "60": 1161062912.0, + "61": 1161062912.0, + "62": 1161062912.0, + "63": 1161062912.0, + "64": 1161062912.0, + "65": 1161062912.0, + "66": 1161062912.0, + "67": 1161062912.0, + "68": 1161062912.0, + "69": 1161062912.0, + "70": 1161062912.0, + "71": 1161062912.0, + "72": 1161062912.0, + "73": 1161062912.0, + "74": 1161062912.0, + "75": 1161062912.0, + "76": 1161062912.0, + "77": 1161062912.0, + "78": 1161062912.0, + "79": 1161062912.0, + "80": 1161062912.0, + "81": 1161062912.0, + "82": 1161062912.0, + "83": 1161062912.0, + "84": 1161062912.0, + "85": 1161062912.0, + "86": 1161062912.0, + "87": 1161062912.0, + "88": 1161062912.0, + "89": 1161062912.0, + "90": 1161062912.0, + "91": 1161062912.0, + "92": 1161062912.0, + "93": 1161062912.0, + "94": 1161062912.0, + "95": 1161062912.0, + "96": 1161062912.0, + "97": 1161062912.0, + "98": 1161062912.0, + "99": 1161062912.0, + "100": 1161062912.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.69857, + "2": 0.5607, + "3": 0.4224, + "4": 0.408, + "5": 0.40601, + "6": 0.40622, + "7": 0.41117, + "8": 0.41277, + "9": 0.41135, + "10": 0.41075, + "11": 0.41036, + "12": 0.41422, + "13": 0.41066, + "14": 0.41137, + "15": 0.41203, + "16": 0.41967, + "17": 0.41058, + "18": 0.41138, + "19": 0.41123, + "20": 0.4101, + "21": 0.40646, + "22": 0.40698, + "23": 0.40933, + "24": 0.40762, + "25": 0.40817, + "26": 0.40815, + "27": 0.40808, + "28": 0.41113, + "29": 0.41068, + "30": 0.40954, + "31": 0.40816, + "32": 0.40856, + "33": 0.40858, + "34": 0.4094, + "35": 0.40789, + "36": 0.40721, + "37": 0.4076, + "38": 0.4074, + "39": 0.40856, + "40": 0.40702, + "41": 0.40749, + "42": 0.40775, + "43": 0.40697, + "44": 0.4055, + "45": 0.40519, + "46": 0.40481, + "47": 0.40587, + "48": 0.40507, + "49": 0.40669, + "50": 0.40562, + "51": 0.42093, + "52": 0.41028, + "53": 0.4088, + "54": 0.41137, + "55": 0.40983, + "56": 0.40893, + "57": 0.41252, + "58": 0.40982, + "59": 0.41034, + "60": 0.41176, + "61": 0.40849, + "62": 0.40923, + "63": 0.40913, + "64": 0.4084, + "65": 0.40845, + "66": 0.41123, + "67": 0.40774, + "68": 0.40592, + "69": 0.4048, + "70": 0.40668, + "71": 0.40553, + "72": 0.40771, + "73": 0.40622, + "74": 0.40899, + "75": 0.41092, + "76": 0.41085, + "77": 0.41068, + "78": 0.4084, + "79": 0.41086, + "80": 0.77469, + "81": 0.41176, + "82": 0.41234, + "83": 0.41225, + "84": 0.41217, + "85": 0.41295, + "86": 0.41174, + "87": 0.40921, + "88": 0.40831, + "89": 0.40924, + "90": 0.40844, + "91": 0.40982, + "92": 0.40955, + "93": 0.40963, + "94": 0.4085, + "95": 0.43734, + "96": 0.40945, + "97": 0.40886, + "98": 0.41034, + "99": 0.40924, + "100": 0.40948 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json index b9747029488..1091699bf9a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86209, "5": 10.88253, "10": 10.83642, "15": 10.82909, "20": 10.73237, "25": 10.56392, "30": 10.38484, "35": 10.28951, "40": 10.09133, "45": 9.83487, "50": 9.91604, "55": 9.88232, "60": 9.51376, "65": 8.95392, "70": 9.73094, "75": 9.4313, "80": 9.40597, "85": 9.61137, "90": 9.81743, "95": 9.51571, "100": 9.40429}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1780.0, "5": 2137.0, "10": 1588.0, "15": 2083.0, "20": 1816.0, "25": 1862.0, "30": 2045.0, "35": 2205.0, "40": 2629.0, "45": 2441.0, "50": 2734.0, "55": 2662.0, "60": 2726.0, "65": 3011.0, "70": 3737.0, "75": 2771.0, "80": 3804.0, "85": 3588.0, "90": 3384.0, "95": 3590.0, "100": 3454.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 368387072.0, "5": 368387072.0, "10": 368387072.0, "15": 368387072.0, "20": 368387072.0, "25": 368387072.0, "30": 368387072.0, "35": 368387072.0, "40": 368387072.0, "45": 368387072.0, "50": 368387072.0, "55": 368387072.0, "60": 368387072.0, "65": 368387072.0, "70": 368387072.0, "75": 368387072.0, "80": 368387072.0, "85": 368387072.0, "90": 368387072.0, "95": 368387072.0, "100": 368387072.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1510972416.0, "5": 1645104640.0, "10": 1645105152.0, "15": 1645105152.0, "20": 1645105152.0, "25": 1645105152.0, "30": 1645105152.0, "35": 1645105152.0, "40": 1645105152.0, "45": 1645105152.0, "50": 1645105152.0, "55": 1645105152.0, "60": 1645105152.0, "65": 1645105152.0, "70": 1645105152.0, "75": 1647199744.0, "80": 1647199744.0, "85": 1647199744.0, "90": 1647199744.0, "95": 1647199744.0, "100": 1647199744.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 9.70786, "5": 0.41822, "10": 0.42132, "15": 0.41057, "20": 0.4105, "25": 0.41988, "30": 0.41035, "35": 0.41599, "40": 0.40546, "45": 0.40496, "50": 0.40588, "55": 0.40982, "60": 0.4157, "65": 0.40455, "70": 0.40547, "75": 0.41128, "80": 0.41661, "85": 0.42011, "90": 0.41367, "95": 0.41851, "100": 0.40235}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85966, + "2": 10.85779, + "3": 10.85806, + "4": 10.85111, + "5": 10.88314, + "6": 10.8873, + "7": 10.86205, + "8": 10.86979, + "9": 10.86489, + "10": 10.83833, + "11": 10.87881, + "12": 10.87319, + "13": 10.87924, + "14": 10.88927, + "15": 10.83002, + "16": 10.83231, + "17": 10.80059, + "18": 10.82546, + "19": 10.81132, + "20": 10.73125, + "21": 10.72292, + "22": 10.57559, + "23": 10.73211, + "24": 10.61836, + "25": 10.56602, + "26": 10.61211, + "27": 10.62224, + "28": 10.58011, + "29": 10.59905, + "30": 10.38311, + "31": 10.14181, + "32": 10.48118, + "33": 10.47322, + "34": 10.23556, + "35": 10.28738, + "36": 10.23493, + "37": 10.35785, + "38": 10.20631, + "39": 10.41397, + "40": 10.09217, + "41": 10.16577, + "42": 10.21856, + "43": 9.85974, + "44": 9.97115, + "45": 9.83404, + "46": 9.84483, + "47": 10.16033, + "48": 9.85135, + "49": 9.53781, + "50": 9.91577, + "51": 9.8567, + "52": 9.75178, + "53": 10.07652, + "54": 9.96084, + "55": 9.88221, + "56": 9.63206, + "57": 9.49147, + "58": 9.83408, + "59": 9.59352, + "60": 9.51388, + "61": 9.69802, + "62": 9.99154, + "63": 9.3723, + "64": 9.77839, + "65": 8.95518, + "66": 9.70976, + "67": 9.38198, + "68": 9.78701, + "69": 9.793, + "70": 9.73033, + "71": 9.61752, + "72": 9.58459, + "73": 9.48958, + "74": 8.94015, + "75": 9.43092, + "76": 9.09168, + "77": 10.06222, + "78": 9.72696, + "79": 9.37408, + "80": 9.40676, + "81": 9.47995, + "82": 9.69225, + "83": 9.31299, + "84": 9.41921, + "85": 9.61096, + "86": 9.06853, + "87": 9.59119, + "88": 9.74582, + "89": 9.60624, + "90": 9.81746, + "91": 9.34247, + "92": 9.35856, + "93": 9.07894, + "94": 8.82753, + "95": 9.51606, + "96": 9.52063, + "97": 9.31097, + "98": 9.67055, + "99": 8.88626, + "100": 9.40485 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1756.0, + "2": 2002.0, + "3": 1950.0, + "4": 1894.0, + "5": 2227.0, + "6": 2168.0, + "7": 2021.0, + "8": 1775.0, + "9": 2051.0, + "10": 1611.0, + "11": 2147.0, + "12": 1965.0, + "13": 2185.0, + "14": 2052.0, + "15": 2134.0, + "16": 2077.0, + "17": 2080.0, + "18": 1914.0, + "19": 1993.0, + "20": 1823.0, + "21": 2142.0, + "22": 1847.0, + "23": 2340.0, + "24": 1831.0, + "25": 1790.0, + "26": 1923.0, + "27": 2066.0, + "28": 2268.0, + "29": 2261.0, + "30": 2043.0, + "31": 1759.0, + "32": 2122.0, + "33": 2389.0, + "34": 2186.0, + "35": 2312.0, + "36": 2176.0, + "37": 2626.0, + "38": 2471.0, + "39": 2624.0, + "40": 2556.0, + "41": 2475.0, + "42": 2504.0, + "43": 2326.0, + "44": 2307.0, + "45": 2355.0, + "46": 2625.0, + "47": 2809.0, + "48": 2621.0, + "49": 2670.0, + "50": 2699.0, + "51": 2829.0, + "52": 2809.0, + "53": 3230.0, + "54": 2864.0, + "55": 2706.0, + "56": 2917.0, + "57": 2529.0, + "58": 3339.0, + "59": 3051.0, + "60": 2623.0, + "61": 3287.0, + "62": 2913.0, + "63": 2639.0, + "64": 3154.0, + "65": 2856.0, + "66": 3465.0, + "67": 2934.0, + "68": 2985.0, + "69": 3298.0, + "70": 3653.0, + "71": 3260.0, + "72": 2684.0, + "73": 3232.0, + "74": 2191.0, + "75": 2766.0, + "76": 3335.0, + "77": 3793.0, + "78": 3608.0, + "79": 3384.0, + "80": 3782.0, + "81": 3969.0, + "82": 3640.0, + "83": 3237.0, + "84": 3606.0, + "85": 3553.0, + "86": 3160.0, + "87": 4130.0, + "88": 3430.0, + "89": 3818.0, + "90": 3363.0, + "91": 3041.0, + "92": 3524.0, + "93": 3060.0, + "94": 3575.0, + "95": 3463.0, + "96": 3921.0, + "97": 3597.0, + "98": 4039.0, + "99": 3435.0, + "100": 3548.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 368387584.0, + "2": 368387584.0, + "3": 368387584.0, + "4": 368387584.0, + "5": 368387584.0, + "6": 368387584.0, + "7": 368387584.0, + "8": 368387584.0, + "9": 368387584.0, + "10": 368387584.0, + "11": 368387584.0, + "12": 368387584.0, + "13": 368387584.0, + "14": 368387584.0, + "15": 368387584.0, + "16": 368387584.0, + "17": 368387584.0, + "18": 368387584.0, + "19": 368387584.0, + "20": 368387584.0, + "21": 368387584.0, + "22": 368387584.0, + "23": 368387584.0, + "24": 368387584.0, + "25": 368387584.0, + "26": 368387584.0, + "27": 368387584.0, + "28": 368387584.0, + "29": 368387584.0, + "30": 368387584.0, + "31": 368387584.0, + "32": 368387584.0, + "33": 368387584.0, + "34": 368387584.0, + "35": 368387584.0, + "36": 368387584.0, + "37": 368387584.0, + "38": 368387584.0, + "39": 368387584.0, + "40": 368387584.0, + "41": 368387584.0, + "42": 368387584.0, + "43": 368387584.0, + "44": 368387584.0, + "45": 368387584.0, + "46": 368387584.0, + "47": 368387584.0, + "48": 368387584.0, + "49": 368387584.0, + "50": 368387584.0, + "51": 368387584.0, + "52": 368387584.0, + "53": 368387584.0, + "54": 368387584.0, + "55": 368387584.0, + "56": 368387584.0, + "57": 368387584.0, + "58": 368387584.0, + "59": 368387584.0, + "60": 368387584.0, + "61": 368387584.0, + "62": 368387584.0, + "63": 368387584.0, + "64": 368387584.0, + "65": 368387584.0, + "66": 368387584.0, + "67": 368387584.0, + "68": 368387584.0, + "69": 368387584.0, + "70": 368387584.0, + "71": 368387584.0, + "72": 368387584.0, + "73": 368387584.0, + "74": 368387584.0, + "75": 368387584.0, + "76": 368387584.0, + "77": 368387584.0, + "78": 368387584.0, + "79": 368387584.0, + "80": 368387584.0, + "81": 368387584.0, + "82": 368387584.0, + "83": 368387584.0, + "84": 368387584.0, + "85": 368387584.0, + "86": 368387584.0, + "87": 368387584.0, + "88": 368387584.0, + "89": 368387584.0, + "90": 368387584.0, + "91": 368387584.0, + "92": 368387584.0, + "93": 368387584.0, + "94": 368387584.0, + "95": 368387584.0, + "96": 368387584.0, + "97": 368387584.0, + "98": 368387584.0, + "99": 368387584.0, + "100": 368387584.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1030075904.0, + "2": 1161062912.0, + "3": 1161062912.0, + "4": 1161062912.0, + "5": 1161062912.0, + "6": 1161062912.0, + "7": 1161062912.0, + "8": 1161062912.0, + "9": 1161062912.0, + "10": 1161062912.0, + "11": 1161062912.0, + "12": 1161062912.0, + "13": 1161062912.0, + "14": 1161062912.0, + "15": 1161062912.0, + "16": 1161062912.0, + "17": 1161062912.0, + "18": 1161062912.0, + "19": 1161062912.0, + "20": 1161062912.0, + "21": 1161062912.0, + "22": 1161062912.0, + "23": 1161062912.0, + "24": 1161062912.0, + "25": 1161062912.0, + "26": 1161062912.0, + "27": 1161062912.0, + "28": 1161062912.0, + "29": 1161062912.0, + "30": 1161062912.0, + "31": 1161062912.0, + "32": 1161062912.0, + "33": 1161062912.0, + "34": 1161062912.0, + "35": 1161062912.0, + "36": 1161062912.0, + "37": 1161062912.0, + "38": 1161062912.0, + "39": 1161062912.0, + "40": 1161062912.0, + "41": 1161062912.0, + "42": 1161062912.0, + "43": 1161062912.0, + "44": 1161062912.0, + "45": 1161062912.0, + "46": 1161062912.0, + "47": 1161062912.0, + "48": 1161062912.0, + "49": 1161062912.0, + "50": 1161062912.0, + "51": 1161062912.0, + "52": 1161062912.0, + "53": 1161062912.0, + "54": 1161062912.0, + "55": 1161062912.0, + "56": 1161062912.0, + "57": 1161062912.0, + "58": 1161062912.0, + "59": 1161062912.0, + "60": 1161062912.0, + "61": 1161062912.0, + "62": 1161062912.0, + "63": 1161062912.0, + "64": 1161062912.0, + "65": 1161062912.0, + "66": 1161062912.0, + "67": 1161062912.0, + "68": 1161062912.0, + "69": 1161062912.0, + "70": 1161062912.0, + "71": 1161062912.0, + "72": 1161062912.0, + "73": 1161062912.0, + "74": 1161062912.0, + "75": 1161062912.0, + "76": 1161062912.0, + "77": 1161062912.0, + "78": 1161062912.0, + "79": 1161062912.0, + "80": 1161062912.0, + "81": 1161062912.0, + "82": 1161062912.0, + "83": 1161062912.0, + "84": 1161062912.0, + "85": 1161062912.0, + "86": 1161062912.0, + "87": 1161062912.0, + "88": 1161062912.0, + "89": 1161062912.0, + "90": 1161062912.0, + "91": 1161062912.0, + "92": 1161062912.0, + "93": 1161062912.0, + "94": 1161062912.0, + "95": 1161062912.0, + "96": 1161062912.0, + "97": 1161062912.0, + "98": 1161062912.0, + "99": 1161062912.0, + "100": 1161062912.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.72553, + "2": 0.52446, + "3": 0.41527, + "4": 0.41699, + "5": 0.41496, + "6": 0.41411, + "7": 0.41829, + "8": 0.41655, + "9": 0.41643, + "10": 0.42008, + "11": 0.41959, + "12": 0.41842, + "13": 0.41485, + "14": 0.41643, + "15": 0.41486, + "16": 0.41617, + "17": 0.41476, + "18": 0.42598, + "19": 0.41821, + "20": 0.41457, + "21": 0.41579, + "22": 0.41438, + "23": 0.41644, + "24": 0.41499, + "25": 0.41537, + "26": 0.41593, + "27": 0.42875, + "28": 0.41636, + "29": 0.41505, + "30": 0.4148, + "31": 0.41806, + "32": 0.41549, + "33": 0.41482, + "34": 0.41559, + "35": 0.4156, + "36": 0.4152, + "37": 0.4152, + "38": 0.4154, + "39": 0.41674, + "40": 0.41745, + "41": 0.41582, + "42": 0.41548, + "43": 0.41428, + "44": 0.4158, + "45": 0.41469, + "46": 0.41584, + "47": 0.41662, + "48": 0.41588, + "49": 0.41548, + "50": 0.42504, + "51": 0.41857, + "52": 0.40985, + "53": 0.40877, + "54": 0.41013, + "55": 0.40869, + "56": 0.84381, + "57": 0.41437, + "58": 0.42502, + "59": 0.41122, + "60": 0.41956, + "61": 0.40996, + "62": 0.40983, + "63": 0.41144, + "64": 0.41126, + "65": 0.41361, + "66": 0.41243, + "67": 0.41431, + "68": 0.4396, + "69": 0.42434, + "70": 0.41269, + "71": 0.42108, + "72": 0.41357, + "73": 0.41116, + "74": 0.41086, + "75": 0.41041, + "76": 0.41106, + "77": 0.41, + "78": 0.41669, + "79": 0.41627, + "80": 0.41237, + "81": 0.41157, + "82": 0.41168, + "83": 0.41229, + "84": 0.41209, + "85": 0.41258, + "86": 0.41294, + "87": 0.41185, + "88": 0.41106, + "89": 0.41159, + "90": 0.41277, + "91": 0.41162, + "92": 0.41309, + "93": 0.41351, + "94": 0.40941, + "95": 0.40961, + "96": 0.41012, + "97": 0.40887, + "98": 0.40809, + "99": 0.40865, + "100": 0.40854 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..128bc5b55bb --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84082, + "2": 10.83544, + "3": 10.83361, + "4": 10.8202, + "5": 10.84212, + "6": 10.87215, + "7": 10.83489, + "8": 10.8434, + "9": 10.84794, + "10": 10.81681, + "11": 10.86063, + "12": 10.85821, + "13": 10.87957, + "14": 10.87773, + "15": 10.85163, + "16": 10.8472, + "17": 10.84071, + "18": 10.86582, + "19": 10.85937, + "20": 10.85131, + "21": 10.85552, + "22": 10.82535, + "23": 10.86895, + "24": 10.82877, + "25": 10.81945, + "26": 10.83385, + "27": 10.82583, + "28": 10.84435, + "29": 10.83124, + "30": 10.75388, + "31": 10.66502, + "32": 10.78819, + "33": 10.76332, + "34": 10.67654, + "35": 10.68319, + "36": 10.63342, + "37": 10.68482, + "38": 10.60112, + "39": 10.69463, + "40": 10.52433, + "41": 10.54208, + "42": 10.56578, + "43": 10.34551, + "44": 10.38873, + "45": 10.31101, + "46": 10.30065, + "47": 10.48062, + "48": 10.28105, + "49": 10.05975, + "50": 10.29413, + "51": 10.23775, + "52": 10.15443, + "53": 10.36085, + "54": 10.26927, + "55": 10.2161, + "56": 9.99594, + "57": 9.8744, + "58": 10.14007, + "59": 9.93447, + "60": 9.84864, + "61": 9.98549, + "62": 10.2164, + "63": 9.69034, + "64": 10.0182, + "65": 9.30046, + "66": 9.9355, + "67": 9.63051, + "68": 9.99128, + "69": 9.9852, + "70": 9.92463, + "71": 9.81436, + "72": 9.79481, + "73": 9.68082, + "74": 9.1945, + "75": 9.60407, + "76": 9.28537, + "77": 10.18507, + "78": 9.86718, + "79": 9.52407, + "80": 9.55749, + "81": 9.62863, + "82": 9.81568, + "83": 9.45708, + "84": 9.53654, + "85": 9.73266, + "86": 9.20138, + "87": 9.69524, + "88": 9.85412, + "89": 9.71648, + "90": 9.91047, + "91": 9.45992, + "92": 9.46603, + "93": 9.19321, + "94": 8.94, + "95": 9.60607, + "96": 9.62214, + "97": 9.39796, + "98": 9.76023, + "99": 8.99097, + "100": 9.49505 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 454770688.0, + "2": 454770688.0, + "3": 454770688.0, + "4": 454770688.0, + "5": 454770688.0, + "6": 454770688.0, + "7": 454770688.0, + "8": 454770688.0, + "9": 454770688.0, + "10": 454770688.0, + "11": 454770688.0, + "12": 454770688.0, + "13": 454770688.0, + "14": 454770688.0, + "15": 454770688.0, + "16": 454770688.0, + "17": 454770688.0, + "18": 518880768.0, + "19": 518880768.0, + "20": 518880768.0, + "21": 518880768.0, + "22": 518880768.0, + "23": 518880768.0, + "24": 518880768.0, + "25": 518880768.0, + "26": 518880768.0, + "27": 518880768.0, + "28": 518880768.0, + "29": 518880768.0, + "30": 518880768.0, + "31": 518880768.0, + "32": 518880768.0, + "33": 518880768.0, + "34": 518880768.0, + "35": 518880768.0, + "36": 518880768.0, + "37": 518880768.0, + "38": 518880768.0, + "39": 518880768.0, + "40": 518880768.0, + "41": 518880768.0, + "42": 518880768.0, + "43": 518880768.0, + "44": 518880768.0, + "45": 518880768.0, + "46": 518880768.0, + "47": 518880768.0, + "48": 518880768.0, + "49": 518880768.0, + "50": 518880768.0, + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, + "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, + "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, + "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, + "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, + "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, + "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, + "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, + "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, + "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, + "100": 518880768.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4511150592.0, + "2": 4544705536.0, + "3": 4544705536.0, + "4": 4544705536.0, + "5": 4544705536.0, + "6": 4544705536.0, + "7": 4544705536.0, + "8": 4544705536.0, + "9": 4544705536.0, + "10": 4544705536.0, + "11": 4544705536.0, + "12": 4544705536.0, + "13": 4544705536.0, + "14": 4544705536.0, + "15": 4544705536.0, + "16": 4544705536.0, + "17": 4544705536.0, + "18": 4544705536.0, + "19": 4607767040.0, + "20": 4607767040.0, + "21": 4607767040.0, + "22": 4607767040.0, + "23": 4607767040.0, + "24": 4607767040.0, + "25": 4607767040.0, + "26": 4607767040.0, + "27": 4607767040.0, + "28": 4607767040.0, + "29": 4607767040.0, + "30": 4607767040.0, + "31": 4607767040.0, + "32": 4607767040.0, + "33": 4607767040.0, + "34": 4607767040.0, + "35": 4607767040.0, + "36": 4607767040.0, + "37": 4607767040.0, + "38": 4607767040.0, + "39": 4607767040.0, + "40": 4607767040.0, + "41": 4607767040.0, + "42": 4607767040.0, + "43": 4607767040.0, + "44": 4607767040.0, + "45": 4607767040.0, + "46": 4607767040.0, + "47": 4607767040.0, + "48": 4607767040.0, + "49": 4607767040.0, + "50": 4607767040.0, + "51": 4607767040.0, + "52": 4607767040.0, + "53": 4607767040.0, + "54": 4607767040.0, + "55": 4607767040.0, + "56": 4607767040.0, + "57": 4607767040.0, + "58": 4607767040.0, + "59": 4607767040.0, + "60": 4607767040.0, + "61": 4607767040.0, + "62": 4607767040.0, + "63": 4607767040.0, + "64": 4607767040.0, + "65": 4607767040.0, + "66": 4607767040.0, + "67": 4607767040.0, + "68": 4607767040.0, + "69": 4607767040.0, + "70": 4607767040.0, + "71": 4607767040.0, + "72": 4607767040.0, + "73": 4607767040.0, + "74": 4607767040.0, + "75": 4607767040.0, + "76": 4607767040.0, + "77": 4607767040.0, + "78": 4607767040.0, + "79": 4607767040.0, + "80": 4607767040.0, + "81": 4607767040.0, + "82": 4607767040.0, + "83": 4607767040.0, + "84": 4607767040.0, + "85": 4607767040.0, + "86": 4607767040.0, + "87": 4607767040.0, + "88": 4607767040.0, + "89": 4607767040.0, + "90": 4607767040.0, + "91": 4607767040.0, + "92": 4607767040.0, + "93": 4607767040.0, + "94": 4607767040.0, + "95": 4607767040.0, + "96": 4607767040.0, + "97": 4607767040.0, + "98": 4607767040.0, + "99": 4607767040.0, + "100": 4607767040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.20459, + "2": 0.14512, + "3": 0.09601, + "4": 0.09045, + "5": 0.09053, + "6": 0.08966, + "7": 0.08974, + "8": 0.08915, + "9": 0.08902, + "10": 0.08919, + "11": 0.08955, + "12": 0.08907, + "13": 0.08947, + "14": 0.08882, + "15": 0.08864, + "16": 0.08897, + "17": 0.08902, + "18": 0.14106, + "19": 0.0977, + "20": 0.09649, + "21": 0.09724, + "22": 0.09168, + "23": 0.09714, + "24": 0.09658, + "25": 0.09708, + "26": 0.09693, + "27": 0.09615, + "28": 0.09697, + "29": 0.09661, + "30": 0.09679, + "31": 0.09716, + "32": 0.09671, + "33": 0.09697, + "34": 0.09691, + "35": 0.09689, + "36": 0.09592, + "37": 0.09711, + "38": 0.09635, + "39": 0.0969, + "40": 0.09642, + "41": 0.09689, + "42": 0.09616, + "43": 0.0971, + "44": 0.09592, + "45": 0.09674, + "46": 0.09586, + "47": 0.09669, + "48": 0.096, + "49": 0.09678, + "50": 0.09589, + "51": 0.09963, + "52": 0.09882, + "53": 0.09639, + "54": 0.09745, + "55": 0.09701, + "56": 0.0971, + "57": 0.0929, + "58": 0.09316, + "59": 0.09353, + "60": 0.0935, + "61": 0.0935, + "62": 0.09059, + "63": 0.09063, + "64": 0.09203, + "65": 0.09133, + "66": 0.09044, + "67": 0.09102, + "68": 0.08852, + "69": 0.08696, + "70": 0.08735, + "71": 0.08724, + "72": 0.08697, + "73": 0.08755, + "74": 0.08731, + "75": 0.08922, + "76": 0.08886, + "77": 0.09297, + "78": 0.0878, + "79": 0.08778, + "80": 0.08751, + "81": 0.08754, + "82": 0.08716, + "83": 0.08674, + "84": 0.0865, + "85": 0.08602, + "86": 0.08613, + "87": 0.08623, + "88": 0.08639, + "89": 0.08691, + "90": 0.08616, + "91": 0.08641, + "92": 0.08645, + "93": 0.08643, + "94": 0.08632, + "95": 0.0862, + "96": 0.08615, + "97": 0.08618, + "98": 0.0866, + "99": 0.0867, + "100": 0.0861 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": 1208.0, + "19": 1541.0, + "20": 1173.0, + "21": 1368.0, + "22": "nan", + "23": 1317.0, + "24": 1131.0, + "25": 1119.0, + "26": 1067.0, + "27": 1300.0, + "28": 1549.0, + "29": 1501.0, + "30": 1354.0, + "31": 1044.0, + "32": 1202.0, + "33": 1503.0, + "34": 1276.0, + "35": 1207.0, + "36": 1217.0, + "37": 1430.0, + "38": 1473.0, + "39": 1525.0, + "40": 1413.0, + "41": 1441.0, + "42": 1487.0, + "43": 1151.0, + "44": 1217.0, + "45": 1314.0, + "46": 1265.0, + "47": 1868.0, + "48": 1314.0, + "49": 1105.0, + "50": 1476.0, + "51": 1508.0, + "52": 1400.0, + "53": 1740.0, + "54": 1498.0, + "55": 1551.0, + "56": 1363.0, + "57": 1465.0, + "58": 1610.0, + "59": 1574.0, + "60": 1599.0, + "61": 1727.0, + "62": 1804.0, + "63": 1590.0, + "64": 1813.0, + "65": 1398.0, + "66": 1738.0, + "67": 1536.0, + "68": 1764.0, + "69": 1781.0, + "70": 1926.0, + "71": 1950.0, + "72": 1461.0, + "73": 1985.0, + "74": 1345.0, + "75": 1871.0, + "76": 1732.0, + "77": 2086.0, + "78": 2075.0, + "79": 1992.0, + "80": 2260.0, + "81": 2300.0, + "82": 2290.0, + "83": 1774.0, + "84": 2172.0, + "85": 2216.0, + "86": 2038.0, + "87": 2741.0, + "88": 2079.0, + "89": 2349.0, + "90": 2315.0, + "91": 1875.0, + "92": 2611.0, + "93": 2053.0, + "94": 2220.0, + "95": 2296.0, + "96": 2665.0, + "97": 2516.0, + "98": 2549.0, + "99": 2378.0, + "100": 2257.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json index a6254c12055..8d29fc96a7f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.84012, - "5": 10.84147, - "10": 10.82107, - "15": 10.85423, - "20": 10.85111, - "25": 10.82229, - "30": 10.75584, - "35": 10.68389, - "40": 10.52756, - "45": 10.3119, - "50": 10.29392, - "55": 10.21882, - "60": 9.8477, - "65": 9.30027, - "70": 9.92473, - "75": 9.60615, - "80": 9.55867, - "85": 9.7331, - "90": 9.91024, - "95": 9.60586, - "100": 9.49453 + "1": 10.84082, + "2": 10.83544, + "3": 10.83361, + "4": 10.8202, + "5": 10.84212, + "6": 10.87215, + "7": 10.83489, + "8": 10.8434, + "9": 10.84794, + "10": 10.81681, + "11": 10.86063, + "12": 10.85821, + "13": 10.87957, + "14": 10.87773, + "15": 10.85163, + "16": 10.8472, + "17": 10.84071, + "18": 10.86582, + "19": 10.85937, + "20": 10.85131, + "21": 10.85552, + "22": 10.82535, + "23": 10.86895, + "24": 10.82877, + "25": 10.81945, + "26": 10.83385, + "27": 10.82583, + "28": 10.84435, + "29": 10.83124, + "30": 10.75388, + "31": 10.66502, + "32": 10.78819, + "33": 10.76332, + "34": 10.67654, + "35": 10.68319, + "36": 10.63342, + "37": 10.68482, + "38": 10.60112, + "39": 10.69463, + "40": 10.52433, + "41": 10.54208, + "42": 10.56578, + "43": 10.34551, + "44": 10.38873, + "45": 10.31101, + "46": 10.30065, + "47": 10.48062, + "48": 10.28105, + "49": 10.05975, + "50": 10.29413, + "51": 10.23775, + "52": 10.15443, + "53": 10.36085, + "54": 10.26927, + "55": 10.2161, + "56": 9.99594, + "57": 9.8744, + "58": 10.14007, + "59": 9.93447, + "60": 9.84864, + "61": 9.98549, + "62": 10.2164, + "63": 9.69034, + "64": 10.0182, + "65": 9.30046, + "66": 9.9355, + "67": 9.63051, + "68": 9.99128, + "69": 9.9852, + "70": 9.92463, + "71": 9.81436, + "72": 9.79481, + "73": 9.68082, + "74": 9.1945, + "75": 9.60407, + "76": 9.28537, + "77": 10.18507, + "78": 9.86718, + "79": 9.52407, + "80": 9.55749, + "81": 9.62863, + "82": 9.81568, + "83": 9.45708, + "84": 9.53654, + "85": 9.73266, + "86": 9.20138, + "87": 9.69524, + "88": 9.85412, + "89": 9.71648, + "90": 9.91047, + "91": 9.45992, + "92": 9.46603, + "93": 9.19321, + "94": 8.94, + "95": 9.60607, + "96": 9.62214, + "97": 9.39796, + "98": 9.76023, + "99": 8.99097, + "100": 9.49505 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 454770688.0, + "2": 454770688.0, + "3": 454770688.0, + "4": 454770688.0, "5": 454770688.0, + "6": 454770688.0, + "7": 454770688.0, + "8": 454770688.0, + "9": 454770688.0, "10": 454770688.0, + "11": 454770688.0, + "12": 454770688.0, + "13": 454770688.0, + "14": 454770688.0, "15": 454770688.0, + "16": 454770688.0, + "17": 454770688.0, + "18": 518880768.0, + "19": 518880768.0, "20": 518880768.0, + "21": 518880768.0, + "22": 518880768.0, + "23": 518880768.0, + "24": 518880768.0, "25": 518880768.0, + "26": 518880768.0, + "27": 518880768.0, + "28": 518880768.0, + "29": 518880768.0, "30": 518880768.0, + "31": 518880768.0, + "32": 518880768.0, + "33": 518880768.0, + "34": 518880768.0, "35": 518880768.0, + "36": 518880768.0, + "37": 518880768.0, + "38": 518880768.0, + "39": 518880768.0, "40": 518880768.0, + "41": 518880768.0, + "42": 518880768.0, + "43": 518880768.0, + "44": 518880768.0, "45": 518880768.0, + "46": 518880768.0, + "47": 518880768.0, + "48": 518880768.0, + "49": 518880768.0, "50": 518880768.0, + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, "100": 518880768.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 4511150592.0, + "2": 4544705536.0, + "3": 4544705536.0, + "4": 4544705536.0, "5": 4544705536.0, + "6": 4544705536.0, + "7": 4544705536.0, + "8": 4544705536.0, + "9": 4544705536.0, "10": 4544705536.0, + "11": 4544705536.0, + "12": 4544705536.0, + "13": 4544705536.0, + "14": 4544705536.0, "15": 4544705536.0, + "16": 4544705536.0, + "17": 4544705536.0, + "18": 4544705536.0, + "19": 4607767040.0, "20": 4607767040.0, + "21": 4607767040.0, + "22": 4607767040.0, + "23": 4607767040.0, + "24": 4607767040.0, "25": 4607767040.0, + "26": 4607767040.0, + "27": 4607767040.0, + "28": 4607767040.0, + "29": 4607767040.0, "30": 4607767040.0, + "31": 4607767040.0, + "32": 4607767040.0, + "33": 4607767040.0, + "34": 4607767040.0, "35": 4607767040.0, + "36": 4607767040.0, + "37": 4607767040.0, + "38": 4607767040.0, + "39": 4607767040.0, "40": 4607767040.0, + "41": 4607767040.0, + "42": 4607767040.0, + "43": 4607767040.0, + "44": 4607767040.0, "45": 4607767040.0, + "46": 4607767040.0, + "47": 4607767040.0, + "48": 4607767040.0, + "49": 4607767040.0, "50": 4607767040.0, + "51": 4607767040.0, + "52": 4607767040.0, + "53": 4607767040.0, + "54": 4607767040.0, "55": 4607767040.0, + "56": 4607767040.0, + "57": 4607767040.0, + "58": 4607767040.0, + "59": 4607767040.0, "60": 4607767040.0, + "61": 4607767040.0, + "62": 4607767040.0, + "63": 4607767040.0, + "64": 4607767040.0, "65": 4607767040.0, + "66": 4607767040.0, + "67": 4607767040.0, + "68": 4607767040.0, + "69": 4607767040.0, "70": 4607767040.0, + "71": 4607767040.0, + "72": 4607767040.0, + "73": 4607767040.0, + "74": 4607767040.0, "75": 4607767040.0, + "76": 4607767040.0, + "77": 4607767040.0, + "78": 4607767040.0, + "79": 4607767040.0, "80": 4607767040.0, + "81": 4607767040.0, + "82": 4607767040.0, + "83": 4607767040.0, + "84": 4607767040.0, "85": 4607767040.0, + "86": 4607767040.0, + "87": 4607767040.0, + "88": 4607767040.0, + "89": 4607767040.0, "90": 4607767040.0, + "91": 4607767040.0, + "92": 4607767040.0, + "93": 4607767040.0, + "94": 4607767040.0, "95": 4607767040.0, + "96": 4607767040.0, + "97": 4607767040.0, + "98": 4607767040.0, + "99": 4607767040.0, "100": 4607767040.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 9.36364, - "5": 0.11207, - "10": 0.09807, - "15": 0.11189, - "20": 0.12039, - "25": 0.10422, - "30": 0.10453, - "35": 0.10312, - "40": 0.10542, - "45": 0.10362, - "50": 0.10448, - "55": 0.10212, - "60": 0.10194, - "65": 0.10259, - "70": 0.10183, - "75": 0.10119, - "80": 0.10042, - "85": 0.10084, - "90": 0.10028, - "95": 0.10004, - "100": 0.09998 + "1": 4.2592, + "2": 0.13544, + "3": 0.09999, + "4": 0.08273, + "5": 0.08157, + "6": 0.08266, + "7": 0.08111, + "8": 0.08184, + "9": 0.08109, + "10": 0.08281, + "11": 0.08041, + "12": 0.08186, + "13": 0.08098, + "14": 0.08513, + "15": 0.0821, + "16": 0.08144, + "17": 0.08052, + "18": 0.13091, + "19": 0.08819, + "20": 0.08804, + "21": 0.08818, + "22": 0.08404, + "23": 0.08729, + "24": 0.08805, + "25": 0.08736, + "26": 0.08811, + "27": 0.08757, + "28": 0.08887, + "29": 0.08961, + "30": 0.0883, + "31": 0.08788, + "32": 0.08884, + "33": 0.08833, + "34": 0.08811, + "35": 0.08831, + "36": 0.08859, + "37": 0.08809, + "38": 0.08879, + "39": 0.08769, + "40": 0.0883, + "41": 0.08757, + "42": 0.08797, + "43": 0.08669, + "44": 0.08751, + "45": 0.08893, + "46": 0.08862, + "47": 0.08698, + "48": 0.089, + "49": 0.08841, + "50": 0.08813, + "51": 0.09282, + "52": 0.08991, + "53": 0.08846, + "54": 0.08878, + "55": 0.08875, + "56": 0.0897, + "57": 0.0888, + "58": 0.08814, + "59": 0.08821, + "60": 0.08782, + "61": 0.0888, + "62": 0.08762, + "63": 0.08743, + "64": 0.0879, + "65": 0.08877, + "66": 0.08656, + "67": 0.08681, + "68": 0.08654, + "69": 0.08705, + "70": 0.08667, + "71": 0.08696, + "72": 0.08664, + "73": 0.08625, + "74": 0.08667, + "75": 0.08656, + "76": 0.08557, + "77": 0.08578, + "78": 0.08586, + "79": 0.08584, + "80": 0.08576, + "81": 0.08653, + "82": 0.08572, + "83": 0.08613, + "84": 0.08557, + "85": 0.08616, + "86": 0.08608, + "87": 0.08563, + "88": 0.08581, + "89": 0.08591, + "90": 0.08593, + "91": 0.08543, + "92": 0.08641, + "93": 0.08635, + "94": 0.08549, + "95": 0.08554, + "96": 0.08487, + "97": 0.08505, + "98": 0.08522, + "99": 0.08533, + "100": 0.08544 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", "15": "nan", - "20": 1143.0, - "25": 1185.0, - "30": 1375.0, + "16": "nan", + "17": "nan", + "18": 1208.0, + "19": 1541.0, + "20": 1173.0, + "21": 1368.0, + "22": "nan", + "23": 1317.0, + "24": 1131.0, + "25": 1119.0, + "26": 1067.0, + "27": 1300.0, + "28": 1549.0, + "29": 1501.0, + "30": 1354.0, + "31": 1044.0, + "32": 1202.0, + "33": 1503.0, + "34": 1276.0, "35": 1207.0, - "40": 1423.0, - "45": 1344.0, - "50": 1559.0, - "55": 1573.0, - "60": 1564.0, - "65": 1496.0, - "70": 1898.0, - "75": 1893.0, - "80": 2134.0, - "85": 2231.0, - "90": 2378.0, - "95": 2285.0, - "100": 2233.0 + "36": 1217.0, + "37": 1430.0, + "38": 1473.0, + "39": 1525.0, + "40": 1413.0, + "41": 1441.0, + "42": 1487.0, + "43": 1151.0, + "44": 1217.0, + "45": 1314.0, + "46": 1265.0, + "47": 1868.0, + "48": 1314.0, + "49": 1105.0, + "50": 1476.0, + "51": 1508.0, + "52": 1400.0, + "53": 1740.0, + "54": 1498.0, + "55": 1551.0, + "56": 1363.0, + "57": 1465.0, + "58": 1610.0, + "59": 1574.0, + "60": 1599.0, + "61": 1727.0, + "62": 1804.0, + "63": 1590.0, + "64": 1813.0, + "65": 1398.0, + "66": 1738.0, + "67": 1536.0, + "68": 1764.0, + "69": 1781.0, + "70": 1926.0, + "71": 1950.0, + "72": 1461.0, + "73": 1985.0, + "74": 1345.0, + "75": 1871.0, + "76": 1732.0, + "77": 2086.0, + "78": 2075.0, + "79": 1992.0, + "80": 2260.0, + "81": 2300.0, + "82": 2290.0, + "83": 1774.0, + "84": 2172.0, + "85": 2216.0, + "86": 2038.0, + "87": 2741.0, + "88": 2079.0, + "89": 2349.0, + "90": 2315.0, + "91": 1875.0, + "92": 2611.0, + "93": 2053.0, + "94": 2220.0, + "95": 2296.0, + "96": 2665.0, + "97": 2516.0, + "98": 2549.0, + "99": 2378.0, + "100": 2257.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json new file mode 100644 index 00000000000..93d8ec41a7b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.83572, + "2": 10.8327, + "3": 10.83034, + "4": 10.80223, + "5": 10.85556, + "6": 10.86667, + "7": 10.84637, + "8": 10.84251, + "9": 10.85429, + "10": 10.80752, + "11": 10.88921, + "12": 10.87043, + "13": 10.87395, + "14": 10.88615, + "15": 10.79982, + "16": 10.81152, + "17": 10.78946, + "18": 10.82078, + "19": 10.80788, + "20": 10.71283, + "21": 10.68647, + "22": 10.54255, + "23": 10.71926, + "24": 10.58463, + "25": 10.54402, + "26": 10.61055, + "27": 10.61018, + "28": 10.57117, + "29": 10.59062, + "30": 10.35312, + "31": 10.0924, + "32": 10.46991, + "33": 10.45536, + "34": 10.19985, + "35": 10.26269, + "36": 10.21197, + "37": 10.33645, + "38": 10.17551, + "39": 10.39309, + "40": 10.07155, + "41": 10.13767, + "42": 10.19633, + "43": 9.81246, + "44": 9.93615, + "45": 9.81584, + "46": 9.80638, + "47": 10.12803, + "48": 9.82444, + "49": 9.50618, + "50": 9.89067 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1564.0, + "2": 1660.0, + "3": 1617.0, + "4": 1711.0, + "5": 1840.0, + "6": 1754.0, + "7": 1793.0, + "8": 1599.0, + "9": 1894.0, + "10": 1424.0, + "11": 1793.0, + "12": 1727.0, + "13": 1856.0, + "14": 1761.0, + "15": 1915.0, + "16": 1859.0, + "17": 1727.0, + "18": 1585.0, + "19": 1747.0, + "20": 1655.0, + "21": 1783.0, + "22": 1671.0, + "23": 2028.0, + "24": 1698.0, + "25": 1618.0, + "26": 1708.0, + "27": 1804.0, + "28": 1978.0, + "29": 1950.0, + "30": 1911.0, + "31": 1582.0, + "32": 1852.0, + "33": 2049.0, + "34": 1912.0, + "35": 1999.0, + "36": 1972.0, + "37": 2236.0, + "38": 2294.0, + "39": 2209.0, + "40": 2171.0, + "41": 2340.0, + "42": 2153.0, + "43": 2015.0, + "44": 2122.0, + "45": 2123.0, + "46": 2194.0, + "47": 2463.0, + "48": 2382.0, + "49": 2300.0, + "50": 2397.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 952847360.0, + "2": 952847360.0, + "3": 952847360.0, + "4": 952847360.0, + "5": 952847360.0, + "6": 952847360.0, + "7": 952847360.0, + "8": 952847360.0, + "9": 952847360.0, + "10": 952847360.0, + "11": 952847360.0, + "12": 952847360.0, + "13": 952847360.0, + "14": 952847360.0, + "15": 952847360.0, + "16": 952847360.0, + "17": 952847360.0, + "18": 952847360.0, + "19": 952847360.0, + "20": 952847360.0, + "21": 952847360.0, + "22": 952847360.0, + "23": 952847360.0, + "24": 952847360.0, + "25": 952847360.0, + "26": 952847360.0, + "27": 952847360.0, + "28": 952847360.0, + "29": 952847360.0, + "30": 952847360.0, + "31": 952847360.0, + "32": 952847360.0, + "33": 952847360.0, + "34": 952847360.0, + "35": 952847360.0, + "36": 952847360.0, + "37": 952847360.0, + "38": 952847360.0, + "39": 952847360.0, + "40": 952847360.0, + "41": 952847360.0, + "42": 952847360.0, + "43": 952847360.0, + "44": 952847360.0, + "45": 952847360.0, + "46": 952847360.0, + "47": 952847360.0, + "48": 952847360.0, + "49": 952847360.0, + "50": 952847360.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3275808768.0, + "2": 3637371904.0, + "3": 3637371904.0, + "4": 3637371904.0, + "5": 3637371904.0, + "6": 3637371904.0, + "7": 3637371904.0, + "8": 3637371904.0, + "9": 3637371904.0, + "10": 3637371904.0, + "11": 3637371904.0, + "12": 3637371904.0, + "13": 3637371904.0, + "14": 3637371904.0, + "15": 3637371904.0, + "16": 3637371904.0, + "17": 3637371904.0, + "18": 3637371904.0, + "19": 3637371904.0, + "20": 3637371904.0, + "21": 3637371904.0, + "22": 3637371904.0, + "23": 3637371904.0, + "24": 3637371904.0, + "25": 3637371904.0, + "26": 3637371904.0, + "27": 3637371904.0, + "28": 3637371904.0, + "29": 3637371904.0, + "30": 3637371904.0, + "31": 3637371904.0, + "32": 3637371904.0, + "33": 3637371904.0, + "34": 3637371904.0, + "35": 3637371904.0, + "36": 3637371904.0, + "37": 3637371904.0, + "38": 3637371904.0, + "39": 3637371904.0, + "40": 3637371904.0, + "41": 3637371904.0, + "42": 3637371904.0, + "43": 3637371904.0, + "44": 3637371904.0, + "45": 3637371904.0, + "46": 3637371904.0, + "47": 3637371904.0, + "48": 3637371904.0, + "49": 3637371904.0, + "50": 3637371904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5.51421, + "2": 0.1601, + "3": 0.13443, + "4": 0.13403, + "5": 0.13289, + "6": 0.13124, + "7": 0.13139, + "8": 0.13164, + "9": 0.12936, + "10": 0.1315, + "11": 0.13031, + "12": 0.12986, + "13": 0.13068, + "14": 0.13035, + "15": 0.12992, + "16": 0.12988, + "17": 0.12998, + "18": 0.13022, + "19": 0.13175, + "20": 0.13299, + "21": 0.13082, + "22": 0.13147, + "23": 0.13039, + "24": 0.13108, + "25": 0.13113, + "26": 0.1301, + "27": 0.12969, + "28": 0.12939, + "29": 0.13157, + "30": 0.13294, + "31": 0.132, + "32": 0.13107, + "33": 0.13092, + "34": 0.12992, + "35": 0.13029, + "36": 0.13044, + "37": 0.13082, + "38": 0.1305, + "39": 0.13098, + "40": 0.13116, + "41": 0.13121, + "42": 0.13136, + "43": 0.13117, + "44": 0.13317, + "45": 0.13163, + "46": 0.1312, + "47": 0.13036, + "48": 0.13105, + "49": 0.13091, + "50": 0.1304 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json index 45cbf767489..bdc8c7f9895 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8337, - "5": 10.85677, - "10": 10.80851, - "15": 10.79856, - "20": 10.71089, - "25": 10.5436, - "30": 10.35069, - "35": 10.26074, + "1": 10.83572, + "2": 10.8327, + "3": 10.83034, + "4": 10.80223, + "5": 10.85556, + "6": 10.86667, + "7": 10.84637, + "8": 10.84251, + "9": 10.85429, + "10": 10.80752, + "11": 10.88921, + "12": 10.87043, + "13": 10.87395, + "14": 10.88615, + "15": 10.79982, + "16": 10.81152, + "17": 10.78946, + "18": 10.82078, + "19": 10.80788, + "20": 10.71283, + "21": 10.68647, + "22": 10.54255, + "23": 10.71926, + "24": 10.58463, + "25": 10.54402, + "26": 10.61055, + "27": 10.61018, + "28": 10.57117, + "29": 10.59062, + "30": 10.35312, + "31": 10.0924, + "32": 10.46991, + "33": 10.45536, + "34": 10.19985, + "35": 10.26269, + "36": 10.21197, + "37": 10.33645, + "38": 10.17551, + "39": 10.39309, "40": 10.07155, - "45": 9.81507, - "50": 9.88952 + "41": 10.13767, + "42": 10.19633, + "43": 9.81246, + "44": 9.93615, + "45": 9.81584, + "46": 9.80638, + "47": 10.12803, + "48": 9.82444, + "49": 9.50618, + "50": 9.89067 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1563.0, - "5": 1808.0, - "10": 1377.0, - "15": 1872.0, - "20": 1675.0, - "25": 1549.0, - "30": 2043.0, - "35": 2002.0, - "40": 2311.0, - "45": 2149.0, - "50": 2368.0 + "1": 1564.0, + "2": 1660.0, + "3": 1617.0, + "4": 1711.0, + "5": 1840.0, + "6": 1754.0, + "7": 1793.0, + "8": 1599.0, + "9": 1894.0, + "10": 1424.0, + "11": 1793.0, + "12": 1727.0, + "13": 1856.0, + "14": 1761.0, + "15": 1915.0, + "16": 1859.0, + "17": 1727.0, + "18": 1585.0, + "19": 1747.0, + "20": 1655.0, + "21": 1783.0, + "22": 1671.0, + "23": 2028.0, + "24": 1698.0, + "25": 1618.0, + "26": 1708.0, + "27": 1804.0, + "28": 1978.0, + "29": 1950.0, + "30": 1911.0, + "31": 1582.0, + "32": 1852.0, + "33": 2049.0, + "34": 1912.0, + "35": 1999.0, + "36": 1972.0, + "37": 2236.0, + "38": 2294.0, + "39": 2209.0, + "40": 2171.0, + "41": 2340.0, + "42": 2153.0, + "43": 2015.0, + "44": 2122.0, + "45": 2123.0, + "46": 2194.0, + "47": 2463.0, + "48": 2382.0, + "49": 2300.0, + "50": 2397.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 948653056.0, - "5": 948653056.0, - "10": 948653056.0, - "15": 948653056.0, - "20": 948653056.0, - "25": 948653056.0, - "30": 948653056.0, - "35": 948653056.0, - "40": 948653056.0, - "45": 948653056.0, - "50": 948653056.0 + "1": 952847360.0, + "2": 952847360.0, + "3": 952847360.0, + "4": 952847360.0, + "5": 952847360.0, + "6": 952847360.0, + "7": 952847360.0, + "8": 952847360.0, + "9": 952847360.0, + "10": 952847360.0, + "11": 952847360.0, + "12": 952847360.0, + "13": 952847360.0, + "14": 952847360.0, + "15": 952847360.0, + "16": 952847360.0, + "17": 952847360.0, + "18": 952847360.0, + "19": 952847360.0, + "20": 952847360.0, + "21": 952847360.0, + "22": 952847360.0, + "23": 952847360.0, + "24": 952847360.0, + "25": 952847360.0, + "26": 952847360.0, + "27": 952847360.0, + "28": 952847360.0, + "29": 952847360.0, + "30": 952847360.0, + "31": 952847360.0, + "32": 952847360.0, + "33": 952847360.0, + "34": 952847360.0, + "35": 952847360.0, + "36": 952847360.0, + "37": 952847360.0, + "38": 952847360.0, + "39": 952847360.0, + "40": 952847360.0, + "41": 952847360.0, + "42": 952847360.0, + "43": 952847360.0, + "44": 952847360.0, + "45": 952847360.0, + "46": 952847360.0, + "47": 952847360.0, + "48": 952847360.0, + "49": 952847360.0, + "50": 952847360.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 3275284480.0, - "5": 3632653312.0, - "10": 3632653312.0, - "15": 3632653312.0, - "20": 3632653312.0, - "25": 3632653312.0, - "30": 3632653312.0, - "35": 3632653312.0, - "40": 3632653312.0, - "45": 3632653312.0, - "50": 3632653312.0 + "1": 3275808768.0, + "2": 3637371904.0, + "3": 3637371904.0, + "4": 3637371904.0, + "5": 3637371904.0, + "6": 3637371904.0, + "7": 3637371904.0, + "8": 3637371904.0, + "9": 3637371904.0, + "10": 3637371904.0, + "11": 3637371904.0, + "12": 3637371904.0, + "13": 3637371904.0, + "14": 3637371904.0, + "15": 3637371904.0, + "16": 3637371904.0, + "17": 3637371904.0, + "18": 3637371904.0, + "19": 3637371904.0, + "20": 3637371904.0, + "21": 3637371904.0, + "22": 3637371904.0, + "23": 3637371904.0, + "24": 3637371904.0, + "25": 3637371904.0, + "26": 3637371904.0, + "27": 3637371904.0, + "28": 3637371904.0, + "29": 3637371904.0, + "30": 3637371904.0, + "31": 3637371904.0, + "32": 3637371904.0, + "33": 3637371904.0, + "34": 3637371904.0, + "35": 3637371904.0, + "36": 3637371904.0, + "37": 3637371904.0, + "38": 3637371904.0, + "39": 3637371904.0, + "40": 3637371904.0, + "41": 3637371904.0, + "42": 3637371904.0, + "43": 3637371904.0, + "44": 3637371904.0, + "45": 3637371904.0, + "46": 3637371904.0, + "47": 3637371904.0, + "48": 3637371904.0, + "49": 3637371904.0, + "50": 3637371904.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.60457, - "5": 0.13759, - "10": 0.13772, - "15": 0.14601, - "20": 0.14893, - "25": 0.14839, - "30": 0.14809, - "35": 0.14777, - "40": 0.14825, - "45": 0.14788, - "50": 0.14852 + "1": 5.86972, + "2": 0.17288, + "3": 0.13781, + "4": 0.13826, + "5": 0.13569, + "6": 0.13252, + "7": 0.1323, + "8": 0.13208, + "9": 0.13237, + "10": 0.13177, + "11": 0.13164, + "12": 0.135, + "13": 0.13389, + "14": 0.13431, + "15": 0.13376, + "16": 0.1342, + "17": 0.13348, + "18": 0.13307, + "19": 0.13389, + "20": 0.13476, + "21": 0.13346, + "22": 0.13333, + "23": 0.13336, + "24": 0.13304, + "25": 0.13373, + "26": 0.13283, + "27": 0.1331, + "28": 0.13314, + "29": 0.13299, + "30": 0.13362, + "31": 0.13392, + "32": 0.13417, + "33": 0.13406, + "34": 0.13351, + "35": 0.13357, + "36": 0.13345, + "37": 0.13422, + "38": 0.1339, + "39": 0.13419, + "40": 0.13437, + "41": 0.13425, + "42": 0.13364, + "43": 0.13389, + "44": 0.13482, + "45": 0.13461, + "46": 0.134, + "47": 0.13363, + "48": 0.13416, + "49": 0.13386, + "50": 0.13343 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..f6017e22e9a --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.83568, + "2": 10.83266, + "3": 10.83151, + "4": 10.80343, + "5": 10.8567, + "6": 10.86778, + "7": 10.84836, + "8": 10.84624, + "9": 10.85924, + "10": 10.81478, + "11": 10.89821, + "12": 10.88433, + "13": 10.88963, + "14": 10.90075, + "15": 10.85098, + "16": 10.86603, + "17": 10.85455, + "18": 10.88507, + "19": 10.8773, + "20": 10.85865, + "21": 10.85654, + "22": 10.79685, + "23": 10.88724, + "24": 10.82649, + "25": 10.81343, + "26": 10.82705, + "27": 10.84612, + "28": 10.84227, + "29": 10.85329, + "30": 10.74969, + "31": 10.63041, + "32": 10.79004, + "33": 10.77234, + "34": 10.65722, + "35": 10.65857, + "36": 10.61583, + "37": 10.67536, + "38": 10.58101, + "39": 10.69083, + "40": 10.50359, + "41": 10.52777, + "42": 10.55371, + "43": 10.28636, + "44": 10.36369, + "45": 10.2738, + "46": 10.24567, + "47": 10.45103, + "48": 10.23707, + "49": 9.99555, + "50": 10.25588, + "51": 10.20129, + "52": 10.10855, + "53": 10.34609, + "54": 10.24857, + "55": 10.18782, + "56": 9.95521, + "57": 9.81221, + "58": 10.10875, + "59": 9.8863, + "60": 9.80901, + "61": 9.94824, + "62": 10.1999, + "63": 9.64431, + "64": 9.9951, + "65": 9.24475, + "66": 9.90917, + "67": 9.59735, + "68": 9.97285, + "69": 9.96332, + "70": 9.91039, + "71": 9.78596, + "72": 9.77263, + "73": 9.6618, + "74": 9.16289, + "75": 9.5812, + "76": 9.26137, + "77": 10.17615, + "78": 9.85644, + "79": 9.50644, + "80": 9.54102, + "81": 9.61313, + "82": 9.80669, + "83": 9.44696, + "84": 9.52782, + "85": 9.72633, + "86": 9.19099, + "87": 9.68736, + "88": 9.85216, + "89": 9.71335, + "90": 9.90316, + "91": 9.46064, + "92": 9.46059, + "93": 9.19418, + "94": 8.93434, + "95": 9.60258, + "96": 9.61852, + "97": 9.39594, + "98": 9.76012, + "99": 8.98668, + "100": 9.49405 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 684610560.0, + "2": 685659136.0, + "3": 685659136.0, + "4": 685659136.0, + "5": 685659136.0, + "6": 685659136.0, + "7": 685659136.0, + "8": 685659136.0, + "9": 685659136.0, + "10": 685659136.0, + "11": 685659136.0, + "12": 685659136.0, + "13": 685659136.0, + "14": 685659136.0, + "15": 685659136.0, + "16": 685659136.0, + "17": 1043027456.0, + "18": 1043027456.0, + "19": 1043027456.0, + "20": 1043027456.0, + "21": 1043027456.0, + "22": 1043027456.0, + "23": 1043027456.0, + "24": 1043027456.0, + "25": 1043027456.0, + "26": 1043027456.0, + "27": 1043027456.0, + "28": 1043027456.0, + "29": 1043027456.0, + "30": 1043027456.0, + "31": 1043027456.0, + "32": 1043027456.0, + "33": 1043027456.0, + "34": 1043027456.0, + "35": 1043027456.0, + "36": 1043027456.0, + "37": 1043027456.0, + "38": 1043027456.0, + "39": 1043027456.0, + "40": 1043027456.0, + "41": 1043027456.0, + "42": 1043027456.0, + "43": 1043027456.0, + "44": 1043027456.0, + "45": 1043027456.0, + "46": 1043027456.0, + "47": 1043027456.0, + "48": 1043027456.0, + "49": 1043027456.0, + "50": 1043027456.0, + "51": 1043027456.0, + "52": 1043027456.0, + "53": 1043027456.0, + "54": 1043027456.0, + "55": 1043027456.0, + "56": 1043027456.0, + "57": 1043027456.0, + "58": 1043027456.0, + "59": 1043027456.0, + "60": 1043027456.0, + "61": 1043027456.0, + "62": 1043027456.0, + "63": 1043027456.0, + "64": 1043027456.0, + "65": 1043027456.0, + "66": 1043027456.0, + "67": 1043027456.0, + "68": 1043027456.0, + "69": 1043027456.0, + "70": 1043027456.0, + "71": 1043027456.0, + "72": 1043027456.0, + "73": 1043027456.0, + "74": 1043027456.0, + "75": 1043027456.0, + "76": 1043027456.0, + "77": 1043027456.0, + "78": 1043027456.0, + "79": 1043027456.0, + "80": 1043027456.0, + "81": 1043027456.0, + "82": 1043027456.0, + "83": 1043027456.0, + "84": 1043027456.0, + "85": 1043027456.0, + "86": 1043027456.0, + "87": 1043027456.0, + "88": 1043027456.0, + "89": 1043027456.0, + "90": 1043027456.0, + "91": 1043027456.0, + "92": 1043027456.0, + "93": 1043027456.0, + "94": 1043027456.0, + "95": 1043027456.0, + "96": 1043027456.0, + "97": 1043027456.0, + "98": 1043027456.0, + "99": 1043027456.0, + "100": 1043027456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3187304960.0, + "2": 3187305472.0, + "3": 3187305472.0, + "4": 3187305472.0, + "5": 3187305472.0, + "6": 3187305472.0, + "7": 3187305472.0, + "8": 3187305472.0, + "9": 3187305472.0, + "10": 3187305472.0, + "11": 3187305472.0, + "12": 3187305472.0, + "13": 3187305472.0, + "14": 3187305472.0, + "15": 3187305472.0, + "16": 3187305472.0, + "17": 3187305472.0, + "18": 3544935936.0, + "19": 3544935936.0, + "20": 3544935936.0, + "21": 3544935936.0, + "22": 3544935936.0, + "23": 3544935936.0, + "24": 3544935936.0, + "25": 3544935936.0, + "26": 3544935936.0, + "27": 3544935936.0, + "28": 3544935936.0, + "29": 3544935936.0, + "30": 3544935936.0, + "31": 3544935936.0, + "32": 3544935936.0, + "33": 3544935936.0, + "34": 3544935936.0, + "35": 3544935936.0, + "36": 3544935936.0, + "37": 3544935936.0, + "38": 3544935936.0, + "39": 3544935936.0, + "40": 3544935936.0, + "41": 3544935936.0, + "42": 3544935936.0, + "43": 3544935936.0, + "44": 3544935936.0, + "45": 3544935936.0, + "46": 3544935936.0, + "47": 3544935936.0, + "48": 3544935936.0, + "49": 3544935936.0, + "50": 3544935936.0, + "51": 3544935936.0, + "52": 3544935936.0, + "53": 3544935936.0, + "54": 3544935936.0, + "55": 3544935936.0, + "56": 3544935936.0, + "57": 3544935936.0, + "58": 3544935936.0, + "59": 3544935936.0, + "60": 3544935936.0, + "61": 3544935936.0, + "62": 3544935936.0, + "63": 3544935936.0, + "64": 3544935936.0, + "65": 3544935936.0, + "66": 3544935936.0, + "67": 3544935936.0, + "68": 3544935936.0, + "69": 3544935936.0, + "70": 3544935936.0, + "71": 3544935936.0, + "72": 3544935936.0, + "73": 3544935936.0, + "74": 3544935936.0, + "75": 3544935936.0, + "76": 3544935936.0, + "77": 3544935936.0, + "78": 3544935936.0, + "79": 3544935936.0, + "80": 3544935936.0, + "81": 3544935936.0, + "82": 3544935936.0, + "83": 3544935936.0, + "84": 3544935936.0, + "85": 3544935936.0, + "86": 3544935936.0, + "87": 3544935936.0, + "88": 3544935936.0, + "89": 3544935936.0, + "90": 3544935936.0, + "91": 3544935936.0, + "92": 3544935936.0, + "93": 3544935936.0, + "94": 3544935936.0, + "95": 3544935936.0, + "96": 3544935936.0, + "97": 3544935936.0, + "98": 3544935936.0, + "99": 3544935936.0, + "100": 3544935936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.71216, + "2": 0.14973, + "3": 0.11823, + "4": 0.11589, + "5": 0.115, + "6": 0.11544, + "7": 0.11494, + "8": 0.11629, + "9": 0.11558, + "10": 0.11559, + "11": 0.11587, + "12": 0.11557, + "13": 0.11565, + "14": 0.11485, + "15": 0.11461, + "16": 0.11592, + "17": 0.17863, + "18": 0.13323, + "19": 0.12474, + "20": 0.12384, + "21": 0.12332, + "22": 0.11528, + "23": 0.12299, + "24": 0.12347, + "25": 0.12391, + "26": 0.12403, + "27": 0.12332, + "28": 0.12373, + "29": 0.12271, + "30": 0.12236, + "31": 0.12244, + "32": 0.12402, + "33": 0.12212, + "34": 0.12202, + "35": 0.1228, + "36": 0.12235, + "37": 0.12236, + "38": 0.12224, + "39": 0.12198, + "40": 0.12302, + "41": 0.12287, + "42": 0.1224, + "43": 0.12222, + "44": 0.12187, + "45": 0.12269, + "46": 0.1224, + "47": 0.12251, + "48": 0.12187, + "49": 0.12239, + "50": 0.12176, + "51": 0.12901, + "52": 0.12249, + "53": 0.12142, + "54": 0.12162, + "55": 0.12146, + "56": 0.12102, + "57": 0.12365, + "58": 0.12178, + "59": 0.12229, + "60": 0.12129, + "61": 0.12124, + "62": 0.12162, + "63": 0.12218, + "64": 0.12145, + "65": 0.1222, + "66": 0.12127, + "67": 0.12238, + "68": 0.12143, + "69": 0.12118, + "70": 0.12095, + "71": 0.12155, + "72": 0.12227, + "73": 0.12265, + "74": 0.1222, + "75": 0.1232, + "76": 0.12175, + "77": 0.12226, + "78": 0.12166, + "79": 0.12163, + "80": 0.12235, + "81": 0.12164, + "82": 0.1221, + "83": 0.12172, + "84": 0.12179, + "85": 0.1221, + "86": 0.12134, + "87": 0.12203, + "88": 0.12166, + "89": 0.12224, + "90": 0.1218, + "91": 0.12135, + "92": 0.12216, + "93": 0.12267, + "94": 0.12146, + "95": 0.1222, + "96": 0.12287, + "97": 0.12296, + "98": 0.12234, + "99": 0.12184, + "100": 0.12221 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2249.0, + "18": 2165.0, + "19": 2362.0, + "20": 1953.0, + "21": 1898.0, + "22": "nan", + "23": 2371.0, + "24": 1984.0, + "25": 1818.0, + "26": 1980.0, + "27": 2078.0, + "28": 2467.0, + "29": 2395.0, + "30": 2298.0, + "31": 1682.0, + "32": 2236.0, + "33": 2192.0, + "34": 1800.0, + "35": 2083.0, + "36": 2139.0, + "37": 2498.0, + "38": 2218.0, + "39": 2642.0, + "40": 2287.0, + "41": 2344.0, + "42": 2304.0, + "43": 2098.0, + "44": 2107.0, + "45": 2243.0, + "46": 1960.0, + "47": 2729.0, + "48": 2418.0, + "49": 1910.0, + "50": 2426.0, + "51": 2335.0, + "52": 2407.0, + "53": 2888.0, + "54": 2477.0, + "55": 2440.0, + "56": 2286.0, + "57": 2340.0, + "58": 2652.0, + "59": 2321.0, + "60": 2493.0, + "61": 2812.0, + "62": 2711.0, + "63": 2367.0, + "64": 2802.0, + "65": 2411.0, + "66": 2869.0, + "67": 2577.0, + "68": 2859.0, + "69": 2524.0, + "70": 3119.0, + "71": 2926.0, + "72": 2251.0, + "73": 2929.0, + "74": 2110.0, + "75": 2884.0, + "76": 2992.0, + "77": 3380.0, + "78": 3484.0, + "79": 3533.0, + "80": 3549.0, + "81": 3616.0, + "82": 3347.0, + "83": 3124.0, + "84": 3276.0, + "85": 3721.0, + "86": 3207.0, + "87": 3941.0, + "88": 3250.0, + "89": 3863.0, + "90": 3452.0, + "91": 2630.0, + "92": 3431.0, + "93": 3123.0, + "94": 3671.0, + "95": 3340.0, + "96": 3874.0, + "97": 3519.0, + "98": 3727.0, + "99": 3447.0, + "100": 3338.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json index 56c78e6ebf6..094be8516a7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -2,141 +2,536 @@ "lm loss": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.8337, - "5": 10.85789, - "10": 10.81578, - "15": 10.84971, - "20": 10.85686, - "25": 10.81326, - "30": 10.74663, - "35": 10.65647, - "40": 10.50351, - "45": 10.27258, - "50": 10.25445, - "55": 10.18699, - "60": 9.8098, - "65": 9.24409, - "70": 9.91024, - "75": 9.58173, - "80": 9.54167, - "85": 9.7262, - "90": 9.90242, - "95": 9.60208, - "100": 9.49424 + "1": 10.83568, + "2": 10.83266, + "3": 10.83151, + "4": 10.80343, + "5": 10.8567, + "6": 10.86778, + "7": 10.84836, + "8": 10.84624, + "9": 10.85924, + "10": 10.81478, + "11": 10.89821, + "12": 10.88433, + "13": 10.88963, + "14": 10.90075, + "15": 10.85098, + "16": 10.86603, + "17": 10.85455, + "18": 10.88507, + "19": 10.8773, + "20": 10.85865, + "21": 10.85654, + "22": 10.79685, + "23": 10.88724, + "24": 10.82649, + "25": 10.81343, + "26": 10.82705, + "27": 10.84612, + "28": 10.84227, + "29": 10.85329, + "30": 10.74969, + "31": 10.63041, + "32": 10.79004, + "33": 10.77234, + "34": 10.65722, + "35": 10.65857, + "36": 10.61583, + "37": 10.67536, + "38": 10.58101, + "39": 10.69083, + "40": 10.50359, + "41": 10.52777, + "42": 10.55371, + "43": 10.28636, + "44": 10.36369, + "45": 10.2738, + "46": 10.24567, + "47": 10.45103, + "48": 10.23707, + "49": 9.99555, + "50": 10.25588, + "51": 10.20129, + "52": 10.10855, + "53": 10.34609, + "54": 10.24857, + "55": 10.18782, + "56": 9.95521, + "57": 9.81221, + "58": 10.10875, + "59": 9.8863, + "60": 9.80901, + "61": 9.94824, + "62": 10.1999, + "63": 9.64431, + "64": 9.9951, + "65": 9.24475, + "66": 9.90917, + "67": 9.59735, + "68": 9.97285, + "69": 9.96332, + "70": 9.91039, + "71": 9.78596, + "72": 9.77263, + "73": 9.6618, + "74": 9.16289, + "75": 9.5812, + "76": 9.26137, + "77": 10.17615, + "78": 9.85644, + "79": 9.50644, + "80": 9.54102, + "81": 9.61313, + "82": 9.80669, + "83": 9.44696, + "84": 9.52782, + "85": 9.72633, + "86": 9.19099, + "87": 9.68736, + "88": 9.85216, + "89": 9.71335, + "90": 9.90316, + "91": 9.46064, + "92": 9.46059, + "93": 9.19418, + "94": 8.93434, + "95": 9.60258, + "96": 9.61852, + "97": 9.39594, + "98": 9.76012, + "99": 8.98668, + "100": 9.49405 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 684610560.0, - "5": 684610560.0, - "10": 684610560.0, - "15": 684610560.0, + "2": 685659136.0, + "3": 685659136.0, + "4": 685659136.0, + "5": 685659136.0, + "6": 685659136.0, + "7": 685659136.0, + "8": 685659136.0, + "9": 685659136.0, + "10": 685659136.0, + "11": 685659136.0, + "12": 685659136.0, + "13": 685659136.0, + "14": 685659136.0, + "15": 685659136.0, + "16": 685659136.0, + "17": 1043027456.0, + "18": 1043027456.0, + "19": 1043027456.0, "20": 1043027456.0, + "21": 1043027456.0, + "22": 1043027456.0, + "23": 1043027456.0, + "24": 1043027456.0, "25": 1043027456.0, + "26": 1043027456.0, + "27": 1043027456.0, + "28": 1043027456.0, + "29": 1043027456.0, "30": 1043027456.0, + "31": 1043027456.0, + "32": 1043027456.0, + "33": 1043027456.0, + "34": 1043027456.0, "35": 1043027456.0, + "36": 1043027456.0, + "37": 1043027456.0, + "38": 1043027456.0, + "39": 1043027456.0, "40": 1043027456.0, + "41": 1043027456.0, + "42": 1043027456.0, + "43": 1043027456.0, + "44": 1043027456.0, "45": 1043027456.0, + "46": 1043027456.0, + "47": 1043027456.0, + "48": 1043027456.0, + "49": 1043027456.0, "50": 1043027456.0, + "51": 1043027456.0, + "52": 1043027456.0, + "53": 1043027456.0, + "54": 1043027456.0, "55": 1043027456.0, + "56": 1043027456.0, + "57": 1043027456.0, + "58": 1043027456.0, + "59": 1043027456.0, "60": 1043027456.0, + "61": 1043027456.0, + "62": 1043027456.0, + "63": 1043027456.0, + "64": 1043027456.0, "65": 1043027456.0, + "66": 1043027456.0, + "67": 1043027456.0, + "68": 1043027456.0, + "69": 1043027456.0, "70": 1043027456.0, + "71": 1043027456.0, + "72": 1043027456.0, + "73": 1043027456.0, + "74": 1043027456.0, "75": 1043027456.0, + "76": 1043027456.0, + "77": 1043027456.0, + "78": 1043027456.0, + "79": 1043027456.0, "80": 1043027456.0, + "81": 1043027456.0, + "82": 1043027456.0, + "83": 1043027456.0, + "84": 1043027456.0, "85": 1043027456.0, + "86": 1043027456.0, + "87": 1043027456.0, + "88": 1043027456.0, + "89": 1043027456.0, "90": 1043027456.0, + "91": 1043027456.0, + "92": 1043027456.0, + "93": 1043027456.0, + "94": 1043027456.0, "95": 1043027456.0, + "96": 1043027456.0, + "97": 1043027456.0, + "98": 1043027456.0, + "99": 1043027456.0, "100": 1043027456.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": 3187304960.0, + "2": 3187305472.0, + "3": 3187305472.0, + "4": 3187305472.0, "5": 3187305472.0, + "6": 3187305472.0, + "7": 3187305472.0, + "8": 3187305472.0, + "9": 3187305472.0, "10": 3187305472.0, + "11": 3187305472.0, + "12": 3187305472.0, + "13": 3187305472.0, + "14": 3187305472.0, "15": 3187305472.0, - "20": 3547033088.0, - "25": 3547033088.0, - "30": 3547033088.0, - "35": 3547033088.0, - "40": 3547033088.0, - "45": 3547033088.0, - "50": 3547033088.0, - "55": 3547033088.0, - "60": 3547033088.0, - "65": 3547033088.0, - "70": 3547033088.0, - "75": 3547033088.0, - "80": 3547033088.0, - "85": 3547033088.0, - "90": 3547033088.0, - "95": 3547033088.0, - "100": 3547033088.0 + "16": 3187305472.0, + "17": 3187305472.0, + "18": 3544935936.0, + "19": 3544935936.0, + "20": 3544935936.0, + "21": 3544935936.0, + "22": 3544935936.0, + "23": 3544935936.0, + "24": 3544935936.0, + "25": 3544935936.0, + "26": 3544935936.0, + "27": 3544935936.0, + "28": 3544935936.0, + "29": 3544935936.0, + "30": 3544935936.0, + "31": 3544935936.0, + "32": 3544935936.0, + "33": 3544935936.0, + "34": 3544935936.0, + "35": 3544935936.0, + "36": 3544935936.0, + "37": 3544935936.0, + "38": 3544935936.0, + "39": 3544935936.0, + "40": 3544935936.0, + "41": 3544935936.0, + "42": 3544935936.0, + "43": 3544935936.0, + "44": 3544935936.0, + "45": 3544935936.0, + "46": 3544935936.0, + "47": 3544935936.0, + "48": 3544935936.0, + "49": 3544935936.0, + "50": 3544935936.0, + "51": 3544935936.0, + "52": 3544935936.0, + "53": 3544935936.0, + "54": 3544935936.0, + "55": 3544935936.0, + "56": 3544935936.0, + "57": 3544935936.0, + "58": 3544935936.0, + "59": 3544935936.0, + "60": 3544935936.0, + "61": 3544935936.0, + "62": 3544935936.0, + "63": 3544935936.0, + "64": 3544935936.0, + "65": 3544935936.0, + "66": 3544935936.0, + "67": 3544935936.0, + "68": 3544935936.0, + "69": 3544935936.0, + "70": 3544935936.0, + "71": 3544935936.0, + "72": 3544935936.0, + "73": 3544935936.0, + "74": 3544935936.0, + "75": 3544935936.0, + "76": 3544935936.0, + "77": 3544935936.0, + "78": 3544935936.0, + "79": 3544935936.0, + "80": 3544935936.0, + "81": 3544935936.0, + "82": 3544935936.0, + "83": 3544935936.0, + "84": 3544935936.0, + "85": 3544935936.0, + "86": 3544935936.0, + "87": 3544935936.0, + "88": 3544935936.0, + "89": 3544935936.0, + "90": 3544935936.0, + "91": 3544935936.0, + "92": 3544935936.0, + "93": 3544935936.0, + "94": 3544935936.0, + "95": 3544935936.0, + "96": 3544935936.0, + "97": 3544935936.0, + "98": 3544935936.0, + "99": 3544935936.0, + "100": 3544935936.0 } }, "iteration-time": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.71226, - "5": 0.14227, - "10": 0.14747, - "15": 0.14757, - "20": 0.1588, - "25": 0.15692, - "30": 0.15748, - "35": 0.15684, - "40": 0.15704, - "45": 0.15798, - "50": 0.15606, - "55": 0.15074, - "60": 0.15245, - "65": 0.15193, - "70": 0.15162, - "75": 0.14856, - "80": 0.14952, - "85": 0.14876, - "90": 0.14933, - "95": 0.15012, - "100": 0.15591 + "1": 7.5468, + "2": 0.1514, + "3": 0.11679, + "4": 0.11442, + "5": 0.11418, + "6": 0.1134, + "7": 0.11341, + "8": 0.11355, + "9": 0.11332, + "10": 0.11336, + "11": 0.11414, + "12": 0.11322, + "13": 0.11309, + "14": 0.11355, + "15": 0.11296, + "16": 0.11311, + "17": 0.19183, + "18": 0.13278, + "19": 0.12368, + "20": 0.1244, + "21": 0.12354, + "22": 0.11533, + "23": 0.12281, + "24": 0.12403, + "25": 0.12406, + "26": 0.12339, + "27": 0.12448, + "28": 0.12265, + "29": 0.1229, + "30": 0.1231, + "31": 0.12325, + "32": 0.12261, + "33": 0.12283, + "34": 0.12275, + "35": 0.12311, + "36": 0.12273, + "37": 0.12367, + "38": 0.12288, + "39": 0.12297, + "40": 0.12264, + "41": 0.1206, + "42": 0.12099, + "43": 0.12152, + "44": 0.12016, + "45": 0.12042, + "46": 0.12101, + "47": 0.12019, + "48": 0.12057, + "49": 0.12054, + "50": 0.12043, + "51": 0.12804, + "52": 0.12188, + "53": 0.12082, + "54": 0.12046, + "55": 0.12243, + "56": 0.12099, + "57": 0.12158, + "58": 0.12118, + "59": 0.12094, + "60": 0.12085, + "61": 0.12158, + "62": 0.12129, + "63": 0.12239, + "64": 0.12127, + "65": 0.12091, + "66": 0.12161, + "67": 0.12115, + "68": 0.12107, + "69": 0.12194, + "70": 0.12208, + "71": 0.12158, + "72": 0.12253, + "73": 0.12311, + "74": 0.12157, + "75": 0.12129, + "76": 0.12243, + "77": 0.1209, + "78": 0.12118, + "79": 0.12236, + "80": 0.12456, + "81": 0.12169, + "82": 0.12201, + "83": 0.12239, + "84": 0.12311, + "85": 0.12253, + "86": 0.12237, + "87": 0.12156, + "88": 0.12306, + "89": 0.12961, + "90": 0.12349, + "91": 0.12189, + "92": 0.12121, + "93": 0.12178, + "94": 0.12615, + "95": 0.12189, + "96": 0.12145, + "97": 0.12112, + "98": 0.12242, + "99": 0.12142, + "100": 0.12094 } }, "num-zeros": { "start_step": 1, "end_step": 100, - "step_interval": 5, + "step_interval": 1, "values": { "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", "15": "nan", - "20": 1969.0, - "25": 1863.0, - "30": 2312.0, - "35": 2063.0, - "40": 2212.0, - "45": 2333.0, - "50": 2369.0, - "55": 2357.0, - "60": 2500.0, - "65": 2438.0, - "70": 3075.0, - "75": 2803.0, - "80": 3568.0, - "85": 3675.0, - "90": 3397.0, - "95": 3468.0, - "100": 3267.0 + "16": "nan", + "17": 2249.0, + "18": 2165.0, + "19": 2362.0, + "20": 1953.0, + "21": 1898.0, + "22": "nan", + "23": 2371.0, + "24": 1984.0, + "25": 1818.0, + "26": 1980.0, + "27": 2078.0, + "28": 2467.0, + "29": 2395.0, + "30": 2298.0, + "31": 1682.0, + "32": 2236.0, + "33": 2192.0, + "34": 1800.0, + "35": 2083.0, + "36": 2139.0, + "37": 2498.0, + "38": 2218.0, + "39": 2642.0, + "40": 2287.0, + "41": 2344.0, + "42": 2304.0, + "43": 2098.0, + "44": 2107.0, + "45": 2243.0, + "46": 1960.0, + "47": 2729.0, + "48": 2418.0, + "49": 1910.0, + "50": 2426.0, + "51": 2335.0, + "52": 2407.0, + "53": 2888.0, + "54": 2477.0, + "55": 2440.0, + "56": 2286.0, + "57": 2340.0, + "58": 2652.0, + "59": 2321.0, + "60": 2493.0, + "61": 2812.0, + "62": 2711.0, + "63": 2367.0, + "64": 2802.0, + "65": 2411.0, + "66": 2869.0, + "67": 2577.0, + "68": 2859.0, + "69": 2524.0, + "70": 3119.0, + "71": 2926.0, + "72": 2251.0, + "73": 2929.0, + "74": 2110.0, + "75": 2884.0, + "76": 2992.0, + "77": 3380.0, + "78": 3484.0, + "79": 3533.0, + "80": 3549.0, + "81": 3616.0, + "82": 3347.0, + "83": 3124.0, + "84": 3276.0, + "85": 3721.0, + "86": 3207.0, + "87": 3941.0, + "88": 3250.0, + "89": 3863.0, + "90": 3452.0, + "91": 2630.0, + "92": 3431.0, + "93": 3123.0, + "94": 3671.0, + "95": 3340.0, + "96": 3874.0, + "97": 3519.0, + "98": 3727.0, + "99": 3447.0, + "100": 3338.0 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json new file mode 100644 index 00000000000..ed50a469580 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79432, + "2": 10.80634, + "3": 10.80586, + "4": 10.77237, + "5": 10.84693, + "6": 10.86605, + "7": 10.82692, + "8": 10.81129, + "9": 10.8342, + "10": 10.76876, + "11": 10.89167, + "12": 10.84477, + "13": 10.85758, + "14": 10.88197, + "15": 10.78831, + "16": 10.7793, + "17": 10.75446, + "18": 10.79296, + "19": 10.79494, + "20": 10.67877, + "21": 10.65858, + "22": 10.50081, + "23": 10.71065, + "24": 10.55089, + "25": 10.50321, + "26": 10.58033, + "27": 10.58262, + "28": 10.55556, + "29": 10.56003, + "30": 10.32992, + "31": 10.08344, + "32": 10.44434, + "33": 10.44238, + "34": 10.19765, + "35": 10.25197, + "36": 10.19117, + "37": 10.31772, + "38": 10.1631, + "39": 10.37486, + "40": 10.05284, + "41": 10.1344, + "42": 10.18877, + "43": 9.80641, + "44": 9.92687, + "45": 9.80332, + "46": 9.811, + "47": 10.12605, + "48": 9.82455, + "49": 9.50975, + "50": 9.88831 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1620.0, + "2": 1664.0, + "3": 1748.0, + "4": 1757.0, + "5": 1754.0, + "6": 1841.0, + "7": 1877.0, + "8": 1697.0, + "9": 1886.0, + "10": 1404.0, + "11": 1889.0, + "12": 1689.0, + "13": 1918.0, + "14": 1757.0, + "15": 1913.0, + "16": 1860.0, + "17": 1797.0, + "18": 1655.0, + "19": 1784.0, + "20": 1616.0, + "21": 1887.0, + "22": 1751.0, + "23": 2100.0, + "24": 1717.0, + "25": 1696.0, + "26": 1723.0, + "27": 1819.0, + "28": 1980.0, + "29": 1962.0, + "30": 2046.0, + "31": 1562.0, + "32": 1935.0, + "33": 2182.0, + "34": 1919.0, + "35": 1994.0, + "36": 1947.0, + "37": 2436.0, + "38": 2218.0, + "39": 2319.0, + "40": 2278.0, + "41": 2348.0, + "42": 2258.0, + "43": 1967.0, + "44": 2011.0, + "45": 2215.0, + "46": 2291.0, + "47": 2519.0, + "48": 2517.0, + "49": 2334.0, + "50": 2325.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 779997184.0, + "2": 779997184.0, + "3": 779997184.0, + "4": 779997184.0, + "5": 779997184.0, + "6": 779997184.0, + "7": 779997184.0, + "8": 779997184.0, + "9": 779997184.0, + "10": 779997184.0, + "11": 779997184.0, + "12": 779997184.0, + "13": 779997184.0, + "14": 779997184.0, + "15": 779997184.0, + "16": 779997184.0, + "17": 779997184.0, + "18": 779997184.0, + "19": 779997184.0, + "20": 779997184.0, + "21": 779997184.0, + "22": 779997184.0, + "23": 779997184.0, + "24": 779997184.0, + "25": 779997184.0, + "26": 779997184.0, + "27": 779997184.0, + "28": 779997184.0, + "29": 779997184.0, + "30": 779997184.0, + "31": 779997184.0, + "32": 779997184.0, + "33": 779997184.0, + "34": 779997184.0, + "35": 779997184.0, + "36": 779997184.0, + "37": 779997184.0, + "38": 779997184.0, + "39": 779997184.0, + "40": 779997184.0, + "41": 779997184.0, + "42": 779997184.0, + "43": 779997184.0, + "44": 779997184.0, + "45": 779997184.0, + "46": 779997184.0, + "47": 779997184.0, + "48": 779997184.0, + "49": 779997184.0, + "50": 779997184.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2463815168.0, + "2": 2746575872.0, + "3": 2746575872.0, + "4": 2746575872.0, + "5": 2746575872.0, + "6": 2746575872.0, + "7": 2746575872.0, + "8": 2746575872.0, + "9": 2746575872.0, + "10": 2746575872.0, + "11": 2746575872.0, + "12": 2746575872.0, + "13": 2746575872.0, + "14": 2746575872.0, + "15": 2746575872.0, + "16": 2746575872.0, + "17": 2746575872.0, + "18": 2746575872.0, + "19": 2746575872.0, + "20": 2746575872.0, + "21": 2746575872.0, + "22": 2746575872.0, + "23": 2746575872.0, + "24": 2746575872.0, + "25": 2746575872.0, + "26": 2746575872.0, + "27": 2746575872.0, + "28": 2746575872.0, + "29": 2746575872.0, + "30": 2746575872.0, + "31": 2746575872.0, + "32": 2746575872.0, + "33": 2746575872.0, + "34": 2746575872.0, + "35": 2746575872.0, + "36": 2746575872.0, + "37": 2746575872.0, + "38": 2746575872.0, + "39": 2746575872.0, + "40": 2746575872.0, + "41": 2746575872.0, + "42": 2746575872.0, + "43": 2746575872.0, + "44": 2746575872.0, + "45": 2746575872.0, + "46": 2746575872.0, + "47": 2746575872.0, + "48": 2746575872.0, + "49": 2746575872.0, + "50": 2746575872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.01512, + "2": 0.18918, + "3": 0.2837, + "4": 0.16138, + "5": 0.1616, + "6": 0.16078, + "7": 0.16042, + "8": 0.15997, + "9": 0.16098, + "10": 0.16102, + "11": 0.16098, + "12": 0.16041, + "13": 0.16091, + "14": 0.16108, + "15": 0.16115, + "16": 0.16297, + "17": 0.16105, + "18": 0.16124, + "19": 0.16111, + "20": 0.16078, + "21": 0.16059, + "22": 0.16143, + "23": 0.1618, + "24": 0.16075, + "25": 0.1607, + "26": 0.16058, + "27": 0.16134, + "28": 0.16032, + "29": 0.16066, + "30": 0.1619, + "31": 0.16069, + "32": 0.16055, + "33": 0.16041, + "34": 0.16109, + "35": 0.1616, + "36": 0.16, + "37": 0.16147, + "38": 0.16094, + "39": 0.16098, + "40": 0.16007, + "41": 0.16179, + "42": 0.1607, + "43": 0.16132, + "44": 0.16157, + "45": 0.16066, + "46": 0.16082, + "47": 0.16056, + "48": 0.16095, + "49": 0.16092, + "50": 0.16103 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json index e7e40a40328..06342d2a540 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts_dgx_a100.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 10.79203, - "5": 10.84593, - "10": 10.76951, - "15": 10.78971, - "20": 10.67886, - "25": 10.50431, - "30": 10.33089, - "35": 10.25278, - "40": 10.05241, - "45": 9.80259, - "50": 9.88804 + "1": 10.79432, + "2": 10.80634, + "3": 10.80586, + "4": 10.77237, + "5": 10.84693, + "6": 10.86605, + "7": 10.82692, + "8": 10.81129, + "9": 10.8342, + "10": 10.76876, + "11": 10.89167, + "12": 10.84477, + "13": 10.85758, + "14": 10.88197, + "15": 10.78831, + "16": 10.7793, + "17": 10.75446, + "18": 10.79296, + "19": 10.79494, + "20": 10.67877, + "21": 10.65858, + "22": 10.50081, + "23": 10.71065, + "24": 10.55089, + "25": 10.50321, + "26": 10.58033, + "27": 10.58262, + "28": 10.55556, + "29": 10.56003, + "30": 10.32992, + "31": 10.08344, + "32": 10.44434, + "33": 10.44238, + "34": 10.19765, + "35": 10.25197, + "36": 10.19117, + "37": 10.31772, + "38": 10.1631, + "39": 10.37486, + "40": 10.05284, + "41": 10.1344, + "42": 10.18877, + "43": 9.80641, + "44": 9.92687, + "45": 9.80332, + "46": 9.811, + "47": 10.12605, + "48": 9.82455, + "49": 9.50975, + "50": 9.88831 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 1649.0, - "5": 1890.0, - "10": 1397.0, - "15": 1928.0, - "20": 1565.0, - "25": 1699.0, - "30": 1943.0, - "35": 2018.0, - "40": 2191.0, - "45": 2207.0, - "50": 2362.0 + "1": 1620.0, + "2": 1664.0, + "3": 1748.0, + "4": 1757.0, + "5": 1754.0, + "6": 1841.0, + "7": 1877.0, + "8": 1697.0, + "9": 1886.0, + "10": 1404.0, + "11": 1889.0, + "12": 1689.0, + "13": 1918.0, + "14": 1757.0, + "15": 1913.0, + "16": 1860.0, + "17": 1797.0, + "18": 1655.0, + "19": 1784.0, + "20": 1616.0, + "21": 1887.0, + "22": 1751.0, + "23": 2100.0, + "24": 1717.0, + "25": 1696.0, + "26": 1723.0, + "27": 1819.0, + "28": 1980.0, + "29": 1962.0, + "30": 2046.0, + "31": 1562.0, + "32": 1935.0, + "33": 2182.0, + "34": 1919.0, + "35": 1994.0, + "36": 1947.0, + "37": 2436.0, + "38": 2218.0, + "39": 2319.0, + "40": 2278.0, + "41": 2348.0, + "42": 2258.0, + "43": 1967.0, + "44": 2011.0, + "45": 2215.0, + "46": 2291.0, + "47": 2519.0, + "48": 2517.0, + "49": 2334.0, + "50": 2325.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 782094336.0, - "5": 782094336.0, - "10": 782094336.0, - "15": 782094336.0, - "20": 782094336.0, - "25": 782094336.0, - "30": 782094336.0, - "35": 782094336.0, - "40": 782094336.0, - "45": 782094336.0, - "50": 782094336.0 + "1": 777900032.0, + "2": 777900032.0, + "3": 777900032.0, + "4": 777900032.0, + "5": 777900032.0, + "6": 777900032.0, + "7": 777900032.0, + "8": 777900032.0, + "9": 777900032.0, + "10": 777900032.0, + "11": 777900032.0, + "12": 777900032.0, + "13": 777900032.0, + "14": 777900032.0, + "15": 777900032.0, + "16": 777900032.0, + "17": 777900032.0, + "18": 777900032.0, + "19": 777900032.0, + "20": 777900032.0, + "21": 777900032.0, + "22": 777900032.0, + "23": 777900032.0, + "24": 777900032.0, + "25": 777900032.0, + "26": 777900032.0, + "27": 777900032.0, + "28": 777900032.0, + "29": 777900032.0, + "30": 777900032.0, + "31": 777900032.0, + "32": 777900032.0, + "33": 777900032.0, + "34": 777900032.0, + "35": 777900032.0, + "36": 777900032.0, + "37": 777900032.0, + "38": 777900032.0, + "39": 777900032.0, + "40": 777900032.0, + "41": 777900032.0, + "42": 777900032.0, + "43": 777900032.0, + "44": 777900032.0, + "45": 777900032.0, + "46": 777900032.0, + "47": 777900032.0, + "48": 777900032.0, + "49": 777900032.0, + "50": 777900032.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 2462767104.0, - "5": 2748673024.0, - "10": 2748673024.0, - "15": 2748673024.0, - "20": 2748673024.0, - "25": 2748673024.0, - "30": 2748673024.0, - "35": 2748673024.0, - "40": 2748673024.0, - "45": 2748673024.0, - "50": 2748673024.0 + "2": 2744478720.0, + "3": 2744478720.0, + "4": 2744478720.0, + "5": 2744478720.0, + "6": 2744478720.0, + "7": 2744478720.0, + "8": 2744478720.0, + "9": 2744478720.0, + "10": 2744478720.0, + "11": 2744478720.0, + "12": 2744478720.0, + "13": 2744478720.0, + "14": 2744478720.0, + "15": 2744478720.0, + "16": 2744478720.0, + "17": 2744478720.0, + "18": 2744478720.0, + "19": 2744478720.0, + "20": 2744478720.0, + "21": 2744478720.0, + "22": 2744478720.0, + "23": 2744478720.0, + "24": 2744478720.0, + "25": 2744478720.0, + "26": 2744478720.0, + "27": 2744478720.0, + "28": 2744478720.0, + "29": 2744478720.0, + "30": 2744478720.0, + "31": 2744478720.0, + "32": 2744478720.0, + "33": 2744478720.0, + "34": 2744478720.0, + "35": 2744478720.0, + "36": 2744478720.0, + "37": 2744478720.0, + "38": 2744478720.0, + "39": 2744478720.0, + "40": 2744478720.0, + "41": 2744478720.0, + "42": 2744478720.0, + "43": 2744478720.0, + "44": 2744478720.0, + "45": 2744478720.0, + "46": 2744478720.0, + "47": 2744478720.0, + "48": 2744478720.0, + "49": 2744478720.0, + "50": 2744478720.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 16.61466, - "5": 0.16864, - "10": 0.16989, - "15": 0.1682, - "20": 0.16954, - "25": 0.16878, - "30": 0.2008, - "35": 0.18359, - "40": 0.16981, - "45": 0.17863, - "50": 0.18407 + "1": 9.69348, + "2": 0.20058, + "3": 0.16793, + "4": 0.16851, + "5": 0.16769, + "6": 0.16776, + "7": 0.1679, + "8": 0.1698, + "9": 0.16773, + "10": 0.16689, + "11": 0.16616, + "12": 0.16649, + "13": 0.16602, + "14": 0.16651, + "15": 0.16681, + "16": 0.16794, + "17": 0.17068, + "18": 0.16616, + "19": 0.16604, + "20": 0.16664, + "21": 0.16675, + "22": 0.16587, + "23": 0.16669, + "24": 0.16593, + "25": 0.16666, + "26": 0.16624, + "27": 0.16546, + "28": 0.16503, + "29": 0.16469, + "30": 0.1651, + "31": 0.16508, + "32": 0.16533, + "33": 0.16475, + "34": 0.16518, + "35": 0.16543, + "36": 0.16422, + "37": 0.1648, + "38": 0.16453, + "39": 0.16423, + "40": 0.16482, + "41": 0.16457, + "42": 0.1653, + "43": 0.16536, + "44": 0.16541, + "45": 0.16481, + "46": 0.16481, + "47": 0.16542, + "48": 0.16607, + "49": 0.1639, + "50": 0.1641 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..44dd2d42336 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79449, + "2": 10.80656, + "3": 10.80727, + "4": 10.77389, + "5": 10.84829, + "6": 10.86736, + "7": 10.82922, + "8": 10.81537, + "9": 10.83956, + "10": 10.77652, + "11": 10.90107, + "12": 10.85927, + "13": 10.87396, + "14": 10.89723, + "15": 10.83961, + "16": 10.83508, + "17": 10.82101, + "18": 10.86029, + "19": 10.86558, + "20": 10.82896, + "21": 10.83275, + "22": 10.75286, + "23": 10.88062, + "24": 10.78219, + "25": 10.76607, + "26": 10.79522, + "27": 10.79866, + "28": 10.81697, + "29": 10.82169, + "30": 10.69891, + "31": 10.55698, + "32": 10.75759, + "33": 10.74362, + "34": 10.59976, + "35": 10.61772, + "36": 10.56389, + "37": 10.63614, + "38": 10.53029, + "39": 10.65358, + "40": 10.44072, + "41": 10.49636, + "42": 10.50954, + "43": 10.22362, + "44": 10.30902, + "45": 10.21065, + "46": 10.19943, + "47": 10.41641, + "48": 10.18128, + "49": 9.94311, + "50": 10.21224, + "51": 10.16759, + "52": 10.06895, + "53": 10.30707, + "54": 10.20911, + "55": 10.15688, + "56": 9.91474, + "57": 9.77696, + "58": 10.07417, + "59": 9.86333, + "60": 9.77328, + "61": 9.9292, + "62": 10.17156, + "63": 9.62041, + "64": 9.97113, + "65": 9.21979, + "66": 9.88693, + "67": 9.58363, + "68": 9.94922, + "69": 9.95271, + "70": 9.89312, + "71": 9.77658, + "72": 9.75435, + "73": 9.6497, + "74": 9.1439, + "75": 9.56121, + "76": 9.25111, + "77": 10.17063, + "78": 9.85402, + "79": 9.49965, + "80": 9.53086, + "81": 9.60555, + "82": 9.80179, + "83": 9.43744, + "84": 9.51987, + "85": 9.7196, + "86": 9.18595, + "87": 9.68687, + "88": 9.8443, + "89": 9.70586, + "90": 9.89977, + "91": 9.45029, + "92": 9.45356, + "93": 9.18554, + "94": 8.92968, + "95": 9.59767, + "96": 9.61491, + "97": 9.39084, + "98": 9.75667, + "99": 8.97921, + "100": 9.49001 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 570640384.0, + "2": 570640384.0, + "3": 570640384.0, + "4": 570640384.0, + "5": 570640384.0, + "6": 570640384.0, + "7": 570640384.0, + "8": 570640384.0, + "9": 570640384.0, + "10": 570640384.0, + "11": 570640384.0, + "12": 570640384.0, + "13": 570640384.0, + "14": 570640384.0, + "15": 570640384.0, + "16": 570640384.0, + "17": 852351488.0, + "18": 852351488.0, + "19": 852351488.0, + "20": 852351488.0, + "21": 852351488.0, + "22": 852351488.0, + "23": 852351488.0, + "24": 852351488.0, + "25": 852351488.0, + "26": 852351488.0, + "27": 852351488.0, + "28": 852351488.0, + "29": 852351488.0, + "30": 852351488.0, + "31": 852351488.0, + "32": 852351488.0, + "33": 852351488.0, + "34": 852351488.0, + "35": 852351488.0, + "36": 852351488.0, + "37": 852351488.0, + "38": 852351488.0, + "39": 852351488.0, + "40": 852351488.0, + "41": 852351488.0, + "42": 852351488.0, + "43": 852351488.0, + "44": 852351488.0, + "45": 852351488.0, + "46": 852351488.0, + "47": 852351488.0, + "48": 852351488.0, + "49": 852351488.0, + "50": 852351488.0, + "51": 852351488.0, + "52": 852351488.0, + "53": 852351488.0, + "54": 852351488.0, + "55": 852351488.0, + "56": 852351488.0, + "57": 852351488.0, + "58": 852351488.0, + "59": 852351488.0, + "60": 852351488.0, + "61": 852351488.0, + "62": 852351488.0, + "63": 852351488.0, + "64": 852351488.0, + "65": 852351488.0, + "66": 852351488.0, + "67": 852351488.0, + "68": 852351488.0, + "69": 852351488.0, + "70": 852351488.0, + "71": 852351488.0, + "72": 852351488.0, + "73": 852351488.0, + "74": 852351488.0, + "75": 852351488.0, + "76": 852351488.0, + "77": 852351488.0, + "78": 852351488.0, + "79": 852351488.0, + "80": 852351488.0, + "81": 852351488.0, + "82": 852351488.0, + "83": 852351488.0, + "84": 852351488.0, + "85": 852351488.0, + "86": 852351488.0, + "87": 852351488.0, + "88": 852351488.0, + "89": 852351488.0, + "90": 852351488.0, + "91": 852351488.0, + "92": 852351488.0, + "93": 852351488.0, + "94": 852351488.0, + "95": 852351488.0, + "96": 852351488.0, + "97": 852351488.0, + "98": 852351488.0, + "99": 852351488.0, + "100": 852351488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2394265600.0, + "2": 2394265600.0, + "3": 2394265600.0, + "4": 2394265600.0, + "5": 2394265600.0, + "6": 2394265600.0, + "7": 2394265600.0, + "8": 2394265600.0, + "9": 2394265600.0, + "10": 2394265600.0, + "11": 2394265600.0, + "12": 2394265600.0, + "13": 2394265600.0, + "14": 2394265600.0, + "15": 2394265600.0, + "16": 2394265600.0, + "17": 2394265600.0, + "18": 2675191296.0, + "19": 2675191296.0, + "20": 2675191296.0, + "21": 2675191296.0, + "22": 2675191296.0, + "23": 2675191296.0, + "24": 2675191296.0, + "25": 2675191296.0, + "26": 2675191296.0, + "27": 2675191296.0, + "28": 2675191296.0, + "29": 2675191296.0, + "30": 2675191296.0, + "31": 2675191296.0, + "32": 2675191296.0, + "33": 2675191296.0, + "34": 2675191296.0, + "35": 2675191296.0, + "36": 2675191296.0, + "37": 2675191296.0, + "38": 2675191296.0, + "39": 2675191296.0, + "40": 2675191296.0, + "41": 2675191296.0, + "42": 2675191296.0, + "43": 2675191296.0, + "44": 2675191296.0, + "45": 2675191296.0, + "46": 2675191296.0, + "47": 2675191296.0, + "48": 2675191296.0, + "49": 2675191296.0, + "50": 2675191296.0, + "51": 2675191296.0, + "52": 2675191296.0, + "53": 2675191296.0, + "54": 2675191296.0, + "55": 2675191296.0, + "56": 2675191296.0, + "57": 2675191296.0, + "58": 2675191296.0, + "59": 2675191296.0, + "60": 2675191296.0, + "61": 2675191296.0, + "62": 2675191296.0, + "63": 2675191296.0, + "64": 2675191296.0, + "65": 2675191296.0, + "66": 2675191296.0, + "67": 2675191296.0, + "68": 2675191296.0, + "69": 2675191296.0, + "70": 2675191296.0, + "71": 2675191296.0, + "72": 2675191296.0, + "73": 2675191296.0, + "74": 2675191296.0, + "75": 2675191296.0, + "76": 2675191296.0, + "77": 2675191296.0, + "78": 2675191296.0, + "79": 2675191296.0, + "80": 2675191296.0, + "81": 2675191296.0, + "82": 2675191296.0, + "83": 2675191296.0, + "84": 2675191296.0, + "85": 2675191296.0, + "86": 2675191296.0, + "87": 2675191296.0, + "88": 2675191296.0, + "89": 2675191296.0, + "90": 2675191296.0, + "91": 2675191296.0, + "92": 2675191296.0, + "93": 2675191296.0, + "94": 2675191296.0, + "95": 2675191296.0, + "96": 2675191296.0, + "97": 2675191296.0, + "98": 2675191296.0, + "99": 2675191296.0, + "100": 2675191296.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.94598, + "2": 0.18624, + "3": 0.25859, + "4": 0.15175, + "5": 0.15224, + "6": 0.15109, + "7": 0.15137, + "8": 0.1508, + "9": 0.1517, + "10": 0.15243, + "11": 0.15119, + "12": 0.15082, + "13": 0.15199, + "14": 0.1522, + "15": 0.15257, + "16": 0.15242, + "17": 0.21211, + "18": 0.16275, + "19": 0.15814, + "20": 0.15774, + "21": 0.15784, + "22": 0.15701, + "23": 0.15663, + "24": 0.15702, + "25": 0.15682, + "26": 0.15759, + "27": 0.15671, + "28": 0.15664, + "29": 0.15666, + "30": 0.15733, + "31": 0.15729, + "32": 0.15598, + "33": 0.1563, + "34": 0.15661, + "35": 0.15747, + "36": 0.15621, + "37": 0.15693, + "38": 0.15652, + "39": 0.15686, + "40": 0.15725, + "41": 0.15693, + "42": 0.1567, + "43": 0.15752, + "44": 0.15682, + "45": 0.15685, + "46": 0.15675, + "47": 0.15696, + "48": 0.15702, + "49": 0.15714, + "50": 0.15648, + "51": 0.1656, + "52": 0.15563, + "53": 0.1545, + "54": 0.15554, + "55": 0.15521, + "56": 0.15534, + "57": 0.15463, + "58": 0.15657, + "59": 0.15534, + "60": 0.15488, + "61": 0.15475, + "62": 0.15571, + "63": 0.15676, + "64": 0.15647, + "65": 0.1567, + "66": 0.15541, + "67": 0.15585, + "68": 0.15577, + "69": 0.15571, + "70": 0.15462, + "71": 0.15484, + "72": 0.15532, + "73": 0.15732, + "74": 0.15634, + "75": 0.15545, + "76": 0.1569, + "77": 0.15472, + "78": 0.15486, + "79": 0.15517, + "80": 0.15532, + "81": 0.15621, + "82": 0.15556, + "83": 0.15556, + "84": 0.15546, + "85": 0.15484, + "86": 0.15503, + "87": 0.15496, + "88": 0.15534, + "89": 0.15573, + "90": 0.15449, + "91": 0.15665, + "92": 0.15529, + "93": 0.15627, + "94": 0.15525, + "95": 0.15483, + "96": 0.15509, + "97": 0.15618, + "98": 0.15569, + "99": 0.15482, + "100": 0.15516 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2437.0, + "18": 2405.0, + "19": 2950.0, + "20": 1827.0, + "21": 2154.0, + "22": 2731.0, + "23": 2609.0, + "24": 2290.0, + "25": 2325.0, + "26": 2079.0, + "27": 2138.0, + "28": 2702.0, + "29": 2576.0, + "30": 2528.0, + "31": 1895.0, + "32": 2628.0, + "33": 2325.0, + "34": 1928.0, + "35": 2061.0, + "36": 2153.0, + "37": 2600.0, + "38": 2350.0, + "39": 2997.0, + "40": 2053.0, + "41": 3352.0, + "42": 2497.0, + "43": 2867.0, + "44": 2109.0, + "45": 2490.0, + "46": 2279.0, + "47": 3051.0, + "48": 2527.0, + "49": 1973.0, + "50": 2887.0, + "51": 2310.0, + "52": 2526.0, + "53": 3705.0, + "54": 2888.0, + "55": 2440.0, + "56": 2496.0, + "57": 2338.0, + "58": 3283.0, + "59": 2849.0, + "60": 2893.0, + "61": 2956.0, + "62": 3134.0, + "63": 3275.0, + "64": 3176.0, + "65": 2318.0, + "66": 3857.0, + "67": 2606.0, + "68": 3313.0, + "69": 2826.0, + "70": 3665.0, + "71": 3011.0, + "72": 2693.0, + "73": 3357.0, + "74": 2271.0, + "75": 2955.0, + "76": 3617.0, + "77": 3936.0, + "78": 3951.0, + "79": 4065.0, + "80": 3665.0, + "81": 5191.0, + "82": 3511.0, + "83": 3263.0, + "84": 3876.0, + "85": 4048.0, + "86": 3414.0, + "87": 3980.0, + "88": 3617.0, + "89": 4400.0, + "90": 3695.0, + "91": 2857.0, + "92": 4432.0, + "93": 3494.0, + "94": 4438.0, + "95": 4076.0, + "96": 3948.0, + "97": 4242.0, + "98": 4943.0, + "99": 3861.0, + "100": 3631.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json index a8c7cb012b6..3b0a03dc6ef 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.79219, "5": 10.84727, "10": 10.77729, "15": 10.84106, "20": 10.82889, "25": 10.7666, "30": 10.69961, "35": 10.61845, "40": 10.44051, "45": 10.20859, "50": 10.21168, "55": 10.15675, "60": 9.77265, "65": 9.22128, "70": 9.89371, "75": 9.56098, "80": 9.5311, "85": 9.71911, "90": 9.89983, "95": 9.59785, "100": 9.49009}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 570640384.0, "5": 570640384.0, "10": 570640384.0, "15": 570640384.0, "20": 852351488.0, "25": 852351488.0, "30": 852351488.0, "35": 852351488.0, "40": 852351488.0, "45": 852351488.0, "50": 852351488.0, "55": 852351488.0, "60": 852351488.0, "65": 852351488.0, "70": 852351488.0, "75": 852351488.0, "80": 852351488.0, "85": 852351488.0, "90": 852351488.0, "95": 852351488.0, "100": 852351488.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2393217536.0, "5": 2393218048.0, "10": 2393218048.0, "15": 2393218048.0, "20": 2675191296.0, "25": 2675191296.0, "30": 2675191296.0, "35": 2675191296.0, "40": 2675191296.0, "45": 2675191296.0, "50": 2675191296.0, "55": 2675191296.0, "60": 2675191296.0, "65": 2675191296.0, "70": 2675191296.0, "75": 2675191296.0, "80": 2675191296.0, "85": 2675191296.0, "90": 2675191296.0, "95": 2675191296.0, "100": 2675191296.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 20.32613, "5": 0.16735, "10": 0.16973, "15": 0.16584, "20": 0.17045, "25": 0.17023, "30": 0.16985, "35": 0.17228, "40": 0.17914, "45": 0.17032, "50": 0.17169, "55": 0.16858, "60": 0.16807, "65": 0.16897, "70": 0.16857, "75": 0.16759, "80": 0.16701, "85": 0.16722, "90": 0.16741, "95": 0.16829, "100": 0.16715}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1788.0, "25": 2300.0, "30": 2419.0, "35": 2097.0, "40": 2126.0, "45": 2520.0, "50": 2929.0, "55": 2457.0, "60": 2961.0, "65": 2359.0, "70": 3633.0, "75": 2932.0, "80": 3595.0, "85": 3959.0, "90": 3785.0, "95": 4072.0, "100": 3581.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79449, + "2": 10.80656, + "3": 10.80727, + "4": 10.77389, + "5": 10.84829, + "6": 10.86736, + "7": 10.82922, + "8": 10.81537, + "9": 10.83956, + "10": 10.77652, + "11": 10.90107, + "12": 10.85927, + "13": 10.87396, + "14": 10.89723, + "15": 10.83961, + "16": 10.83508, + "17": 10.82101, + "18": 10.86029, + "19": 10.86558, + "20": 10.82896, + "21": 10.83275, + "22": 10.75286, + "23": 10.88062, + "24": 10.78219, + "25": 10.76607, + "26": 10.79522, + "27": 10.79866, + "28": 10.81697, + "29": 10.82169, + "30": 10.69891, + "31": 10.55698, + "32": 10.75759, + "33": 10.74362, + "34": 10.59976, + "35": 10.61772, + "36": 10.56389, + "37": 10.63614, + "38": 10.53029, + "39": 10.65358, + "40": 10.44072, + "41": 10.49636, + "42": 10.50954, + "43": 10.22362, + "44": 10.30902, + "45": 10.21065, + "46": 10.19943, + "47": 10.41641, + "48": 10.18128, + "49": 9.94311, + "50": 10.21224, + "51": 10.16759, + "52": 10.06895, + "53": 10.30707, + "54": 10.20911, + "55": 10.15688, + "56": 9.91474, + "57": 9.77696, + "58": 10.07417, + "59": 9.86333, + "60": 9.77328, + "61": 9.9292, + "62": 10.17156, + "63": 9.62041, + "64": 9.97113, + "65": 9.21979, + "66": 9.88693, + "67": 9.58363, + "68": 9.94922, + "69": 9.95271, + "70": 9.89312, + "71": 9.77658, + "72": 9.75435, + "73": 9.6497, + "74": 9.1439, + "75": 9.56121, + "76": 9.25111, + "77": 10.17063, + "78": 9.85402, + "79": 9.49965, + "80": 9.53086, + "81": 9.60555, + "82": 9.80179, + "83": 9.43744, + "84": 9.51987, + "85": 9.7196, + "86": 9.18595, + "87": 9.68687, + "88": 9.8443, + "89": 9.70586, + "90": 9.89977, + "91": 9.45029, + "92": 9.45356, + "93": 9.18554, + "94": 8.92968, + "95": 9.59767, + "96": 9.61491, + "97": 9.39084, + "98": 9.75667, + "99": 8.97921, + "100": 9.49001 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 570640384.0, + "2": 570640384.0, + "3": 570640384.0, + "4": 570640384.0, + "5": 570640384.0, + "6": 570640384.0, + "7": 570640384.0, + "8": 570640384.0, + "9": 570640384.0, + "10": 570640384.0, + "11": 570640384.0, + "12": 570640384.0, + "13": 570640384.0, + "14": 570640384.0, + "15": 570640384.0, + "16": 570640384.0, + "17": 852351488.0, + "18": 852351488.0, + "19": 852351488.0, + "20": 852351488.0, + "21": 852351488.0, + "22": 852351488.0, + "23": 852351488.0, + "24": 852351488.0, + "25": 852351488.0, + "26": 852351488.0, + "27": 852351488.0, + "28": 852351488.0, + "29": 852351488.0, + "30": 852351488.0, + "31": 852351488.0, + "32": 852351488.0, + "33": 852351488.0, + "34": 852351488.0, + "35": 852351488.0, + "36": 852351488.0, + "37": 852351488.0, + "38": 852351488.0, + "39": 852351488.0, + "40": 852351488.0, + "41": 852351488.0, + "42": 852351488.0, + "43": 852351488.0, + "44": 852351488.0, + "45": 852351488.0, + "46": 852351488.0, + "47": 852351488.0, + "48": 852351488.0, + "49": 852351488.0, + "50": 852351488.0, + "51": 852351488.0, + "52": 852351488.0, + "53": 852351488.0, + "54": 852351488.0, + "55": 852351488.0, + "56": 852351488.0, + "57": 852351488.0, + "58": 852351488.0, + "59": 852351488.0, + "60": 852351488.0, + "61": 852351488.0, + "62": 852351488.0, + "63": 852351488.0, + "64": 852351488.0, + "65": 852351488.0, + "66": 852351488.0, + "67": 852351488.0, + "68": 852351488.0, + "69": 852351488.0, + "70": 852351488.0, + "71": 852351488.0, + "72": 852351488.0, + "73": 852351488.0, + "74": 852351488.0, + "75": 852351488.0, + "76": 852351488.0, + "77": 852351488.0, + "78": 852351488.0, + "79": 852351488.0, + "80": 852351488.0, + "81": 852351488.0, + "82": 852351488.0, + "83": 852351488.0, + "84": 852351488.0, + "85": 852351488.0, + "86": 852351488.0, + "87": 852351488.0, + "88": 852351488.0, + "89": 852351488.0, + "90": 852351488.0, + "91": 852351488.0, + "92": 852351488.0, + "93": 852351488.0, + "94": 852351488.0, + "95": 852351488.0, + "96": 852351488.0, + "97": 852351488.0, + "98": 852351488.0, + "99": 852351488.0, + "100": 852351488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2393217536.0, + "2": 2393218048.0, + "3": 2393218048.0, + "4": 2394266624.0, + "5": 2394266624.0, + "6": 2394266624.0, + "7": 2394266624.0, + "8": 2394266624.0, + "9": 2394266624.0, + "10": 2394266624.0, + "11": 2394266624.0, + "12": 2394266624.0, + "13": 2394266624.0, + "14": 2394266624.0, + "15": 2394266624.0, + "16": 2394266624.0, + "17": 2394266624.0, + "18": 2675191296.0, + "19": 2675191296.0, + "20": 2675191296.0, + "21": 2675191296.0, + "22": 2675191296.0, + "23": 2675191296.0, + "24": 2675191296.0, + "25": 2675191296.0, + "26": 2675191296.0, + "27": 2675191296.0, + "28": 2675191296.0, + "29": 2675191296.0, + "30": 2675191296.0, + "31": 2675191296.0, + "32": 2675191296.0, + "33": 2675191296.0, + "34": 2675191296.0, + "35": 2675191296.0, + "36": 2675191296.0, + "37": 2675191296.0, + "38": 2675191296.0, + "39": 2675191296.0, + "40": 2675191296.0, + "41": 2675191296.0, + "42": 2675191296.0, + "43": 2675191296.0, + "44": 2675191296.0, + "45": 2675191296.0, + "46": 2675191296.0, + "47": 2675191296.0, + "48": 2675191296.0, + "49": 2675191296.0, + "50": 2675191296.0, + "51": 2675191296.0, + "52": 2675191296.0, + "53": 2675191296.0, + "54": 2675191296.0, + "55": 2675191296.0, + "56": 2675191296.0, + "57": 2675191296.0, + "58": 2675191296.0, + "59": 2675191296.0, + "60": 2675191296.0, + "61": 2675191296.0, + "62": 2675191296.0, + "63": 2675191296.0, + "64": 2675191296.0, + "65": 2675191296.0, + "66": 2675191296.0, + "67": 2675191296.0, + "68": 2675191296.0, + "69": 2675191296.0, + "70": 2675191296.0, + "71": 2675191296.0, + "72": 2675191296.0, + "73": 2675191296.0, + "74": 2675191296.0, + "75": 2675191296.0, + "76": 2675191296.0, + "77": 2675191296.0, + "78": 2675191296.0, + "79": 2675191296.0, + "80": 2675191296.0, + "81": 2675191296.0, + "82": 2675191296.0, + "83": 2675191296.0, + "84": 2675191296.0, + "85": 2675191296.0, + "86": 2675191296.0, + "87": 2675191296.0, + "88": 2675191296.0, + "89": 2675191296.0, + "90": 2675191296.0, + "91": 2675191296.0, + "92": 2675191296.0, + "93": 2675191296.0, + "94": 2675191296.0, + "95": 2675191296.0, + "96": 2675191296.0, + "97": 2675191296.0, + "98": 2675191296.0, + "99": 2675191296.0, + "100": 2675191296.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.58025, + "2": 0.18555, + "3": 0.31194, + "4": 0.1522, + "5": 0.15205, + "6": 0.1496, + "7": 0.14979, + "8": 0.14921, + "9": 0.14957, + "10": 0.15024, + "11": 0.14887, + "12": 0.14852, + "13": 0.14925, + "14": 0.15079, + "15": 0.14925, + "16": 0.14936, + "17": 0.2057, + "18": 0.15996, + "19": 0.15397, + "20": 0.15414, + "21": 0.1543, + "22": 0.15499, + "23": 0.15504, + "24": 0.15679, + "25": 0.15462, + "26": 0.15509, + "27": 0.15394, + "28": 0.15487, + "29": 0.15522, + "30": 0.1553, + "31": 0.15536, + "32": 0.15406, + "33": 0.15461, + "34": 0.1548, + "35": 0.15472, + "36": 0.15413, + "37": 0.1548, + "38": 0.15446, + "39": 0.15545, + "40": 0.15442, + "41": 0.15567, + "42": 0.15413, + "43": 0.15585, + "44": 0.15428, + "45": 0.15497, + "46": 0.15438, + "47": 0.15508, + "48": 0.15481, + "49": 0.15466, + "50": 0.15476, + "51": 0.16245, + "52": 0.15411, + "53": 0.15376, + "54": 0.15405, + "55": 0.15375, + "56": 0.15402, + "57": 0.15434, + "58": 0.15404, + "59": 0.15454, + "60": 0.15434, + "61": 0.15384, + "62": 0.15505, + "63": 0.15431, + "64": 0.15388, + "65": 0.1547, + "66": 0.15453, + "67": 0.15364, + "68": 0.15388, + "69": 0.15362, + "70": 0.15366, + "71": 0.15425, + "72": 0.15393, + "73": 0.15476, + "74": 0.15414, + "75": 0.15415, + "76": 0.1535, + "77": 0.15481, + "78": 0.1541, + "79": 0.15382, + "80": 0.15363, + "81": 0.15386, + "82": 0.18555, + "83": 0.15422, + "84": 0.15393, + "85": 0.15462, + "86": 0.15512, + "87": 0.15391, + "88": 0.15431, + "89": 0.15431, + "90": 0.15521, + "91": 0.15475, + "92": 0.154, + "93": 0.15414, + "94": 0.15426, + "95": 0.15422, + "96": 0.15393, + "97": 0.15497, + "98": 0.1538, + "99": 0.15481, + "100": 0.15442 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2437.0, + "18": 2405.0, + "19": 2950.0, + "20": 1827.0, + "21": 2154.0, + "22": 2731.0, + "23": 2609.0, + "24": 2290.0, + "25": 2325.0, + "26": 2079.0, + "27": 2138.0, + "28": 2702.0, + "29": 2576.0, + "30": 2528.0, + "31": 1895.0, + "32": 2628.0, + "33": 2325.0, + "34": 1928.0, + "35": 2061.0, + "36": 2153.0, + "37": 2600.0, + "38": 2350.0, + "39": 2997.0, + "40": 2053.0, + "41": 3352.0, + "42": 2497.0, + "43": 2867.0, + "44": 2109.0, + "45": 2490.0, + "46": 2279.0, + "47": 3051.0, + "48": 2527.0, + "49": 1973.0, + "50": 2887.0, + "51": 2310.0, + "52": 2526.0, + "53": 3705.0, + "54": 2888.0, + "55": 2440.0, + "56": 2496.0, + "57": 2338.0, + "58": 3283.0, + "59": 2849.0, + "60": 2893.0, + "61": 2956.0, + "62": 3134.0, + "63": 3275.0, + "64": 3176.0, + "65": 2318.0, + "66": 3857.0, + "67": 2606.0, + "68": 3313.0, + "69": 2826.0, + "70": 3665.0, + "71": 3011.0, + "72": 2693.0, + "73": 3357.0, + "74": 2271.0, + "75": 2955.0, + "76": 3617.0, + "77": 3936.0, + "78": 3951.0, + "79": 4065.0, + "80": 3665.0, + "81": 5191.0, + "82": 3511.0, + "83": 3263.0, + "84": 3876.0, + "85": 4048.0, + "86": 3414.0, + "87": 3980.0, + "88": 3617.0, + "89": 4400.0, + "90": 3695.0, + "91": 2857.0, + "92": 4432.0, + "93": 3494.0, + "94": 4438.0, + "95": 4076.0, + "96": 3948.0, + "97": 4242.0, + "98": 4943.0, + "99": 3861.0, + "100": 3631.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json new file mode 100644 index 00000000000..646419f3d75 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465367040.0, + "4": 1465367040.0, + "5": 1465367040.0, + "6": 1465367040.0, + "7": 1465368064.0, + "8": 1465368064.0, + "9": 1465368064.0, + "10": 1465368064.0, + "11": 1465368064.0, + "12": 1465368064.0, + "13": 1465368064.0, + "14": 1465368064.0, + "15": 1465368064.0, + "16": 1465368064.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.93622, + "2": 0.31349, + "3": 0.28163, + "4": 0.27921, + "5": 0.27918, + "6": 0.27813, + "7": 0.27823, + "8": 0.27889, + "9": 0.27839, + "10": 0.2785, + "11": 0.27567, + "12": 0.27474, + "13": 0.2746, + "14": 0.27424, + "15": 0.27387, + "16": 0.4724, + "17": 0.27409, + "18": 0.28601, + "19": 0.28651, + "20": 0.28586, + "21": 0.29036, + "22": 0.27749, + "23": 0.29042, + "24": 0.29143, + "25": 0.29191, + "26": 0.29136, + "27": 0.29223, + "28": 0.29253, + "29": 0.29127, + "30": 0.29158, + "31": 0.29283, + "32": 0.29224, + "33": 0.29404, + "34": 0.29214, + "35": 0.29125, + "36": 0.29141, + "37": 0.29119, + "38": 0.29131, + "39": 0.29195, + "40": 0.29134, + "41": 0.29104, + "42": 0.2909, + "43": 0.29202, + "44": 0.29149, + "45": 0.29285, + "46": 0.29117, + "47": 0.29278, + "48": 0.29328, + "49": 0.2921, + "50": 0.29144, + "51": 0.2946, + "52": 0.29041, + "53": 0.28952, + "54": 0.28982, + "55": 0.28907, + "56": 0.28812, + "57": 0.2882, + "58": 0.29098, + "59": 0.29394, + "60": 0.29381, + "61": 0.29366, + "62": 0.29495, + "63": 0.29248, + "64": 0.29429, + "65": 0.29369, + "66": 0.29315, + "67": 0.29284, + "68": 0.2941, + "69": 0.29349, + "70": 0.29458, + "71": 0.29385, + "72": 0.29691, + "73": 0.29214, + "74": 0.292, + "75": 0.29157, + "76": 0.29169, + "77": 0.29162, + "78": 0.2922, + "79": 0.29243, + "80": 0.29113, + "81": 0.2927, + "82": 0.29109, + "83": 0.29115, + "84": 0.29064, + "85": 0.29302, + "86": 0.29203, + "87": 0.29135, + "88": 0.29306, + "89": 0.29406, + "90": 0.29439, + "91": 0.29348, + "92": 0.30393, + "93": 0.29287, + "94": 0.29232, + "95": 0.29711, + "96": 0.29103, + "97": 0.29472, + "98": 0.29334, + "99": 0.29389, + "100": 0.29184 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json index c8abc0e775c..9f83249318a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86104, "5": 10.88371, "10": 10.84263, "15": 10.87935, "20": 10.87405, "25": 10.82867, "30": 10.77191, "35": 10.67622, "40": 10.517, "45": 10.28436, "50": 10.27861, "55": 10.20112, "60": 9.83306, "65": 9.26979, "70": 9.92662, "75": 9.61385, "80": 9.56419, "85": 9.74319, "90": 9.92148, "95": 9.6163, "100": 9.5087}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 284527616.0, "5": 284527616.0, "10": 284527616.0, "15": 284527616.0, "20": 416513536.0, "25": 416513536.0, "30": 416513536.0, "35": 416513536.0, "40": 416513536.0, "45": 416513536.0, "50": 416513536.0, "55": 416513536.0, "60": 416513536.0, "65": 416513536.0, "70": 416513536.0, "75": 416513536.0, "80": 416513536.0, "85": 416513536.0, "90": 416513536.0, "95": 416513536.0, "100": 416513536.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1465368064.0, "5": 1465368576.0, "10": 1465368576.0, "15": 1465368576.0, "20": 1596305408.0, "25": 1596305920.0, "30": 1596305920.0, "35": 1596305920.0, "40": 1596305920.0, "45": 1596305920.0, "50": 1596305920.0, "55": 1596305920.0, "60": 1596305920.0, "65": 1596305920.0, "70": 1596305920.0, "75": 1596305920.0, "80": 1596305920.0, "85": 1596305920.0, "90": 1596305920.0, "95": 1596305920.0, "100": 1596305920.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 7.32922, "5": 0.3111, "10": 0.3117, "15": 0.30486, "20": 0.3229, "25": 0.31791, "30": 0.31805, "35": 0.32249, "40": 0.32048, "45": 0.32173, "50": 0.31691, "55": 0.31145, "60": 0.31713, "65": 0.31381, "70": 0.31308, "75": 0.31799, "80": 0.31492, "85": 0.31756, "90": 0.31282, "95": 0.41456, "100": 0.30993}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1777.0, "25": 2124.0, "30": 2262.0, "35": 1763.0, "40": 1875.0, "45": 2434.0, "50": 2634.0, "55": 2383.0, "60": 2832.0, "65": 2223.0, "70": 3836.0, "75": 2913.0, "80": 3398.0, "85": 3618.0, "90": 3676.0, "95": 4082.0, "100": 3502.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367552.0, + "3": 1465367552.0, + "4": 1465367552.0, + "5": 1465367552.0, + "6": 1465367552.0, + "7": 1465367552.0, + "8": 1465367552.0, + "9": 1465367552.0, + "10": 1465367552.0, + "11": 1465367552.0, + "12": 1465367552.0, + "13": 1465367552.0, + "14": 1465367552.0, + "15": 1465367552.0, + "16": 1465367552.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.81628, + "2": 0.32142, + "3": 0.27555, + "4": 0.28299, + "5": 0.28901, + "6": 0.28043, + "7": 0.29138, + "8": 0.30944, + "9": 0.28461, + "10": 0.28789, + "11": 0.28709, + "12": 0.29186, + "13": 0.29114, + "14": 0.29464, + "15": 0.31626, + "16": 0.48847, + "17": 0.28436, + "18": 0.30264, + "19": 0.29287, + "20": 0.30599, + "21": 0.29335, + "22": 0.27957, + "23": 0.29491, + "24": 0.29371, + "25": 0.29398, + "26": 0.29344, + "27": 0.29457, + "28": 0.29449, + "29": 0.29412, + "30": 0.29337, + "31": 0.29404, + "32": 0.29391, + "33": 0.29483, + "34": 0.29389, + "35": 0.29433, + "36": 0.29449, + "37": 0.29463, + "38": 0.29428, + "39": 0.29385, + "40": 0.29379, + "41": 0.29345, + "42": 0.29404, + "43": 0.29413, + "44": 0.29357, + "45": 0.29308, + "46": 0.29302, + "47": 0.29311, + "48": 0.29341, + "49": 0.2946, + "50": 0.29365, + "51": 0.29978, + "52": 0.31599, + "53": 0.29361, + "54": 0.29341, + "55": 0.29321, + "56": 0.29262, + "57": 0.29474, + "58": 0.29427, + "59": 0.29281, + "60": 0.29314, + "61": 0.29219, + "62": 0.29346, + "63": 0.29348, + "64": 0.30211, + "65": 0.29324, + "66": 0.29357, + "67": 0.29314, + "68": 0.29229, + "69": 0.30197, + "70": 0.29329, + "71": 0.30206, + "72": 0.29435, + "73": 0.29495, + "74": 0.2943, + "75": 0.29926, + "76": 0.29332, + "77": 0.29464, + "78": 0.29342, + "79": 0.29434, + "80": 0.29439, + "81": 0.29391, + "82": 0.29436, + "83": 0.29426, + "84": 0.29408, + "85": 0.29452, + "86": 0.29406, + "87": 0.29421, + "88": 0.29373, + "89": 0.29437, + "90": 0.29425, + "91": 0.29383, + "92": 0.2933, + "93": 0.29369, + "94": 0.2937, + "95": 0.29465, + "96": 0.29439, + "97": 0.29435, + "98": 0.2952, + "99": 0.29361, + "100": 0.2936 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..a7feec6d207 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465368576.0, + "3": 1465368576.0, + "4": 1465368576.0, + "5": 1465368576.0, + "6": 1465368576.0, + "7": 1465368576.0, + "8": 1465368576.0, + "9": 1465368576.0, + "10": 1465368576.0, + "11": 1465368576.0, + "12": 1465368576.0, + "13": 1465368576.0, + "14": 1465368576.0, + "15": 1465368576.0, + "16": 1465368576.0, + "17": 1597092352.0, + "18": 1597484544.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.79491, + "2": 0.31619, + "3": 0.28095, + "4": 0.28167, + "5": 0.28111, + "6": 0.28053, + "7": 0.27397, + "8": 0.27382, + "9": 0.2729, + "10": 0.28341, + "11": 0.27301, + "12": 0.27739, + "13": 0.27433, + "14": 0.27589, + "15": 0.27377, + "16": 0.45757, + "17": 0.28453, + "18": 0.29428, + "19": 0.29061, + "20": 0.28748, + "21": 0.28947, + "22": 0.27305, + "23": 0.28802, + "24": 0.29052, + "25": 0.28857, + "26": 0.28938, + "27": 0.28789, + "28": 0.29629, + "29": 0.29633, + "30": 0.29452, + "31": 0.29612, + "32": 0.29638, + "33": 0.29802, + "34": 0.29681, + "35": 0.29807, + "36": 0.29648, + "37": 0.29732, + "38": 0.29691, + "39": 0.29854, + "40": 0.29738, + "41": 0.29825, + "42": 0.29748, + "43": 0.29734, + "44": 0.29746, + "45": 0.29836, + "46": 0.29887, + "47": 0.29739, + "48": 0.29926, + "49": 0.29788, + "50": 0.29895, + "51": 0.29828, + "52": 0.29254, + "53": 0.29242, + "54": 0.29234, + "55": 0.29132, + "56": 0.29158, + "57": 0.29153, + "58": 0.29113, + "59": 0.29239, + "60": 0.29622, + "61": 0.29477, + "62": 0.29358, + "63": 0.29465, + "64": 0.29445, + "65": 0.29533, + "66": 0.29531, + "67": 0.29354, + "68": 0.29434, + "69": 0.29379, + "70": 0.29468, + "71": 0.29423, + "72": 0.29484, + "73": 0.29584, + "74": 0.2949, + "75": 0.29561, + "76": 0.29586, + "77": 0.29535, + "78": 0.29491, + "79": 0.29562, + "80": 0.29515, + "81": 0.2951, + "82": 0.29484, + "83": 0.2928, + "84": 0.29076, + "85": 0.29104, + "86": 0.29208, + "87": 0.29116, + "88": 0.29384, + "89": 0.29252, + "90": 0.29125, + "91": 0.29184, + "92": 0.29214, + "93": 0.29151, + "94": 0.29183, + "95": 0.29202, + "96": 0.29138, + "97": 0.29213, + "98": 0.2924, + "99": 0.29154, + "100": 0.29129 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json index f9728255608..0eef09cf2c1 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts_dgx_a100.json @@ -1 +1,537 @@ -{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.86104, "5": 10.88371, "10": 10.84263, "15": 10.87935, "20": 10.87405, "25": 10.82867, "30": 10.77191, "35": 10.67622, "40": 10.517, "45": 10.28436, "50": 10.27861, "55": 10.20112, "60": 9.83306, "65": 9.26979, "70": 9.92662, "75": 9.61385, "80": 9.56419, "85": 9.74319, "90": 9.92148, "95": 9.6163, "100": 9.5087}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 284527616.0, "5": 284527616.0, "10": 284527616.0, "15": 284527616.0, "20": 416513536.0, "25": 416513536.0, "30": 416513536.0, "35": 416513536.0, "40": 416513536.0, "45": 416513536.0, "50": 416513536.0, "55": 416513536.0, "60": 416513536.0, "65": 416513536.0, "70": 416513536.0, "75": 416513536.0, "80": 416513536.0, "85": 416513536.0, "90": 416513536.0, "95": 416513536.0, "100": 416513536.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1465368064.0, "5": 1465368576.0, "10": 1465368576.0, "15": 1465368576.0, "20": 1596305920.0, "25": 1596305920.0, "30": 1596305920.0, "35": 1596305920.0, "40": 1596305920.0, "45": 1596305920.0, "50": 1596305920.0, "55": 1596305920.0, "60": 1596305920.0, "65": 1596305920.0, "70": 1596305920.0, "75": 1596305920.0, "80": 1596305920.0, "85": 1596305920.0, "90": 1596305920.0, "95": 1596305920.0, "100": 1596305920.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 8.79594, "5": 0.30836, "10": 0.30237, "15": 0.30482, "20": 0.31834, "25": 0.31761, "30": 0.31732, "35": 0.31511, "40": 0.31755, "45": 0.31516, "50": 0.31689, "55": 0.31106, "60": 0.31323, "65": 0.31051, "70": 0.31046, "75": 0.31201, "80": 0.30466, "85": 0.3063, "90": 0.30526, "95": 0.30371, "100": 0.30515}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 1777.0, "25": 2124.0, "30": 2262.0, "35": 1763.0, "40": 1875.0, "45": 2434.0, "50": 2634.0, "55": 2383.0, "60": 2832.0, "65": 2223.0, "70": 3836.0, "75": 2913.0, "80": 3398.0, "85": 3618.0, "90": 3676.0, "95": 4082.0, "100": 3502.0}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465368064.0, + "4": 1465368576.0, + "5": 1465368576.0, + "6": 1465368576.0, + "7": 1465368576.0, + "8": 1465368576.0, + "9": 1465368576.0, + "10": 1465368576.0, + "11": 1465368576.0, + "12": 1465368576.0, + "13": 1465368576.0, + "14": 1465368576.0, + "15": 1465368576.0, + "16": 1465368576.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.02782, + "2": 0.31435, + "3": 0.27957, + "4": 0.27933, + "5": 0.27866, + "6": 0.27855, + "7": 0.2779, + "8": 0.27621, + "9": 0.27704, + "10": 0.27611, + "11": 0.27501, + "12": 0.27489, + "13": 0.27468, + "14": 0.27386, + "15": 0.27315, + "16": 0.41595, + "17": 0.27523, + "18": 0.28979, + "19": 0.28871, + "20": 0.2888, + "21": 0.28867, + "22": 0.27653, + "23": 0.29205, + "24": 0.29078, + "25": 0.29104, + "26": 0.29087, + "27": 0.28794, + "28": 0.28784, + "29": 0.28659, + "30": 0.28669, + "31": 0.28638, + "32": 0.2878, + "33": 0.28717, + "34": 0.28616, + "35": 0.28626, + "36": 0.28648, + "37": 0.28977, + "38": 0.28615, + "39": 0.2864, + "40": 0.28588, + "41": 0.28749, + "42": 0.28735, + "43": 0.28605, + "44": 0.28798, + "45": 0.2882, + "46": 0.28727, + "47": 0.28616, + "48": 0.28603, + "49": 0.2876, + "50": 0.29155, + "51": 0.30309, + "52": 0.29889, + "53": 0.29736, + "54": 0.29772, + "55": 0.29611, + "56": 0.29565, + "57": 0.29413, + "58": 0.29391, + "59": 0.29344, + "60": 0.29428, + "61": 0.29695, + "62": 0.29282, + "63": 0.29418, + "64": 0.29352, + "65": 0.29274, + "66": 0.29449, + "67": 0.29627, + "68": 0.29636, + "69": 0.29393, + "70": 0.28967, + "71": 0.28925, + "72": 0.28962, + "73": 0.28944, + "74": 0.28948, + "75": 0.28996, + "76": 0.28938, + "77": 0.28855, + "78": 0.28891, + "79": 0.28905, + "80": 0.28968, + "81": 0.28873, + "82": 0.28966, + "83": 0.2884, + "84": 0.28842, + "85": 0.29077, + "86": 0.28927, + "87": 0.28888, + "88": 0.28909, + "89": 0.28807, + "90": 0.28887, + "91": 0.28894, + "92": 0.28908, + "93": 0.28985, + "94": 0.289, + "95": 0.28861, + "96": 0.28831, + "97": 0.2877, + "98": 0.29019, + "99": 0.28839, + "100": 0.2881 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..d547773495d --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81189, + "2": 10.82837, + "3": 10.81793, + "4": 10.80617, + "5": 10.84927, + "6": 10.86922, + "7": 10.84362, + "8": 10.83419, + "9": 10.84988, + "10": 10.78408, + "11": 10.88038, + "12": 10.85362, + "13": 10.86619, + "14": 10.88223, + "15": 10.82311, + "16": 10.81052, + "17": 10.78202, + "18": 10.81541, + "19": 10.81756, + "20": 10.74584, + "21": 10.72627, + "22": 10.60022, + "23": 10.74769, + "24": 10.6365, + "25": 10.59453, + "26": 10.64799, + "27": 10.66378, + "28": 10.64562, + "29": 10.65242, + "30": 10.44192, + "31": 10.21776, + "32": 10.52786, + "33": 10.52222, + "34": 10.30567, + "35": 10.35142, + "36": 10.32502, + "37": 10.42585, + "38": 10.28522, + "39": 10.47826, + "40": 10.19414, + "41": 10.22953, + "42": 10.29398, + "43": 9.95703, + "44": 10.06609, + "45": 9.95939, + "46": 9.92711, + "47": 10.22774, + "48": 9.96376, + "49": 9.65724, + "50": 10.01644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31381.0, + "2": 33910.0, + "3": 33570.0, + "4": 31384.0, + "5": 35903.0, + "6": 37599.0, + "7": 35525.0, + "8": 31635.0, + "9": 34509.0, + "10": 30410.0, + "11": 37757.0, + "12": 35967.0, + "13": 36886.0, + "14": 38227.0, + "15": 35788.0, + "16": 36247.0, + "17": 35378.0, + "18": 35284.0, + "19": 36213.0, + "20": 33124.0, + "21": 33424.0, + "22": 31395.0, + "23": 38414.0, + "24": 32277.0, + "25": 31634.0, + "26": 35264.0, + "27": 35935.0, + "28": 37638.0, + "29": 38145.0, + "30": 33261.0, + "31": 30972.0, + "32": 36646.0, + "33": 38318.0, + "34": 33243.0, + "35": 34813.0, + "36": 35446.0, + "37": 38278.0, + "38": 35590.0, + "39": 38681.0, + "40": 36027.0, + "41": 36509.0, + "42": 37301.0, + "43": 34597.0, + "44": 34024.0, + "45": 35938.0, + "46": 37229.0, + "47": 40862.0, + "48": 35537.0, + "49": 35095.0, + "50": 39711.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 880441344.0, + "2": 880437760.0, + "3": 880442368.0, + "4": 880448512.0, + "5": 880440832.0, + "6": 880441344.0, + "7": 880445440.0, + "8": 880442880.0, + "9": 880440320.0, + "10": 880443392.0, + "11": 880442368.0, + "12": 880439808.0, + "13": 880439808.0, + "14": 880443904.0, + "15": 880443904.0, + "16": 880441856.0, + "17": 880448512.0, + "18": 880437760.0, + "19": 880440320.0, + "20": 880442880.0, + "21": 880445440.0, + "22": 880450048.0, + "23": 880449024.0, + "24": 880440320.0, + "25": 880440320.0, + "26": 880445952.0, + "27": 880441344.0, + "28": 880439808.0, + "29": 880443392.0, + "30": 880441344.0, + "31": 880454656.0, + "32": 880444928.0, + "33": 880440320.0, + "34": 880440832.0, + "35": 880446976.0, + "36": 880441344.0, + "37": 880442880.0, + "38": 880439808.0, + "39": 880441856.0, + "40": 880442880.0, + "41": 880445952.0, + "42": 880441344.0, + "43": 880444416.0, + "44": 880439808.0, + "45": 880440832.0, + "46": 880447488.0, + "47": 880440320.0, + "48": 880441856.0, + "49": 880444416.0, + "50": 880442368.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2909851648.0, + "2": 3151821824.0, + "3": 3152806912.0, + "4": 3156619264.0, + "5": 3156619264.0, + "6": 3156619264.0, + "7": 3156619264.0, + "8": 3156619264.0, + "9": 3156619264.0, + "10": 3156619264.0, + "11": 3156619264.0, + "12": 3156619264.0, + "13": 3156619264.0, + "14": 3156619264.0, + "15": 3156619264.0, + "16": 3156619264.0, + "17": 3156619264.0, + "18": 3156619264.0, + "19": 3156619264.0, + "20": 3156619264.0, + "21": 3156619264.0, + "22": 3156619264.0, + "23": 3156619264.0, + "24": 3156619264.0, + "25": 3156619264.0, + "26": 3156619264.0, + "27": 3156619264.0, + "28": 3156619264.0, + "29": 3156619264.0, + "30": 3156619264.0, + "31": 3167942656.0, + "32": 3167942656.0, + "33": 3167942656.0, + "34": 3167942656.0, + "35": 3167942656.0, + "36": 3167942656.0, + "37": 3167942656.0, + "38": 3167942656.0, + "39": 3167942656.0, + "40": 3167942656.0, + "41": 3167942656.0, + "42": 3167942656.0, + "43": 3167942656.0, + "44": 3167942656.0, + "45": 3167942656.0, + "46": 3167942656.0, + "47": 3167942656.0, + "48": 3167942656.0, + "49": 3167942656.0, + "50": 3167942656.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.71086, + "2": 0.34876, + "3": 0.30494, + "4": 0.2983, + "5": 0.31094, + "6": 0.29581, + "7": 0.29535, + "8": 0.29392, + "9": 0.31226, + "10": 0.29284, + "11": 0.29297, + "12": 0.29417, + "13": 0.29411, + "14": 0.29617, + "15": 0.30137, + "16": 0.29884, + "17": 0.29723, + "18": 0.2965, + "19": 0.29582, + "20": 0.29827, + "21": 0.29422, + "22": 0.29464, + "23": 0.29372, + "24": 0.29476, + "25": 0.29448, + "26": 0.29512, + "27": 0.29393, + "28": 0.2949, + "29": 0.29363, + "30": 0.29407, + "31": 0.29478, + "32": 0.2959, + "33": 0.29578, + "34": 0.29539, + "35": 0.295, + "36": 0.29765, + "37": 0.29552, + "38": 0.2962, + "39": 0.29593, + "40": 0.29517, + "41": 0.29558, + "42": 0.2974, + "43": 0.29483, + "44": 0.29357, + "45": 0.29143, + "46": 0.29287, + "47": 0.28977, + "48": 0.30711, + "49": 0.29644, + "50": 0.29278 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json index 63ffa2aa706..f91ad30ed3a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.8028, "5": 10.85317, "10": 10.79253, "15": 10.81762, "20": 10.74263, "25": 10.58726, "30": 10.43425, "35": 10.35057, "40": 10.18634, "45": 9.95891, "50": 10.0194}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 31321.0, "5": 36637.0, "10": 30318.0, "15": 35096.0, "20": 33013.0, "25": 30677.0, "30": 32775.0, "35": 34420.0, "40": 36074.0, "45": 35624.0, "50": 39159.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 863505920.0, "5": 863502848.0, "10": 863507456.0, "15": 863503872.0, "20": 863504896.0, "25": 863506944.0, "30": 863508992.0, "35": 863511040.0, "40": 863508480.0, "45": 863503872.0, "50": 863505920.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2741182976.0, "5": 2984047104.0, "10": 2984047104.0, "15": 2984047104.0, "20": 2985508864.0, "25": 2985508864.0, "30": 2985508864.0, "35": 2986774016.0, "40": 2988336640.0, "45": 2988336640.0, "50": 2990856704.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 30.77131, "5": 0.27562, "10": 0.2823, "15": 0.2732, "20": 0.2737, "25": 0.28195, "30": 0.27083, "35": 0.27437, "40": 0.26787, "45": 0.26819, "50": 0.28362}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81189, + "2": 10.82837, + "3": 10.81793, + "4": 10.80617, + "5": 10.84927, + "6": 10.86922, + "7": 10.84362, + "8": 10.83419, + "9": 10.84988, + "10": 10.78408, + "11": 10.88038, + "12": 10.85362, + "13": 10.86619, + "14": 10.88223, + "15": 10.82311, + "16": 10.81052, + "17": 10.78202, + "18": 10.81541, + "19": 10.81756, + "20": 10.74584, + "21": 10.72627, + "22": 10.60022, + "23": 10.74769, + "24": 10.6365, + "25": 10.59453, + "26": 10.64799, + "27": 10.66378, + "28": 10.64562, + "29": 10.65242, + "30": 10.44192, + "31": 10.21776, + "32": 10.52786, + "33": 10.52222, + "34": 10.30567, + "35": 10.35142, + "36": 10.32502, + "37": 10.42585, + "38": 10.28522, + "39": 10.47826, + "40": 10.19414, + "41": 10.22953, + "42": 10.29398, + "43": 9.95703, + "44": 10.06609, + "45": 9.95939, + "46": 9.92711, + "47": 10.22774, + "48": 9.96376, + "49": 9.65724, + "50": 10.01644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31381.0, + "2": 33910.0, + "3": 33570.0, + "4": 31384.0, + "5": 35903.0, + "6": 37599.0, + "7": 35525.0, + "8": 31635.0, + "9": 34509.0, + "10": 30410.0, + "11": 37757.0, + "12": 35967.0, + "13": 36886.0, + "14": 38227.0, + "15": 35788.0, + "16": 36247.0, + "17": 35378.0, + "18": 35284.0, + "19": 36213.0, + "20": 33124.0, + "21": 33424.0, + "22": 31395.0, + "23": 38414.0, + "24": 32277.0, + "25": 31634.0, + "26": 35264.0, + "27": 35935.0, + "28": 37638.0, + "29": 38145.0, + "30": 33261.0, + "31": 30972.0, + "32": 36646.0, + "33": 38318.0, + "34": 33243.0, + "35": 34813.0, + "36": 35446.0, + "37": 38278.0, + "38": 35590.0, + "39": 38681.0, + "40": 36027.0, + "41": 36509.0, + "42": 37301.0, + "43": 34597.0, + "44": 34024.0, + "45": 35938.0, + "46": 37229.0, + "47": 40862.0, + "48": 35537.0, + "49": 35095.0, + "50": 39711.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 880441344.0, + "2": 880437760.0, + "3": 880442368.0, + "4": 880448512.0, + "5": 880440832.0, + "6": 880441344.0, + "7": 880445440.0, + "8": 880442880.0, + "9": 880440320.0, + "10": 880443392.0, + "11": 880442368.0, + "12": 880439808.0, + "13": 880439808.0, + "14": 880443904.0, + "15": 880443904.0, + "16": 880441856.0, + "17": 880448512.0, + "18": 880437760.0, + "19": 880440320.0, + "20": 880442880.0, + "21": 880445440.0, + "22": 880450048.0, + "23": 880449024.0, + "24": 880440320.0, + "25": 880440320.0, + "26": 880445952.0, + "27": 880441344.0, + "28": 880439808.0, + "29": 880443392.0, + "30": 880441344.0, + "31": 880454656.0, + "32": 880444928.0, + "33": 880440320.0, + "34": 880440832.0, + "35": 880446976.0, + "36": 880441344.0, + "37": 880442880.0, + "38": 880439808.0, + "39": 880441856.0, + "40": 880442880.0, + "41": 880445952.0, + "42": 880441344.0, + "43": 880444416.0, + "44": 880439808.0, + "45": 880440832.0, + "46": 880447488.0, + "47": 880440320.0, + "48": 880441856.0, + "49": 880444416.0, + "50": 880442368.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2910859264.0, + "2": 3151821824.0, + "3": 3152806912.0, + "4": 3156619264.0, + "5": 3156619264.0, + "6": 3156619264.0, + "7": 3156619264.0, + "8": 3156619264.0, + "9": 3156619264.0, + "10": 3156619264.0, + "11": 3156619264.0, + "12": 3156619264.0, + "13": 3156619264.0, + "14": 3156619264.0, + "15": 3156619264.0, + "16": 3156619264.0, + "17": 3156619264.0, + "18": 3156619264.0, + "19": 3156619264.0, + "20": 3156619264.0, + "21": 3156619264.0, + "22": 3156619264.0, + "23": 3156619264.0, + "24": 3156619264.0, + "25": 3156619264.0, + "26": 3156619264.0, + "27": 3156619264.0, + "28": 3156619264.0, + "29": 3156619264.0, + "30": 3156619264.0, + "31": 3167942656.0, + "32": 3167942656.0, + "33": 3167942656.0, + "34": 3167942656.0, + "35": 3167942656.0, + "36": 3167942656.0, + "37": 3167942656.0, + "38": 3167942656.0, + "39": 3167942656.0, + "40": 3167942656.0, + "41": 3167942656.0, + "42": 3167942656.0, + "43": 3167942656.0, + "44": 3167942656.0, + "45": 3167942656.0, + "46": 3167942656.0, + "47": 3167942656.0, + "48": 3167942656.0, + "49": 3167942656.0, + "50": 3167942656.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.90142, + "2": 0.35609, + "3": 0.29589, + "4": 0.29327, + "5": 0.29594, + "6": 0.293, + "7": 0.29087, + "8": 0.29178, + "9": 0.29184, + "10": 0.29303, + "11": 0.29381, + "12": 0.29249, + "13": 0.2936, + "14": 0.29671, + "15": 0.29969, + "16": 0.30214, + "17": 0.29463, + "18": 0.30986, + "19": 0.29429, + "20": 0.29497, + "21": 0.29609, + "22": 0.29421, + "23": 0.2931, + "24": 0.29341, + "25": 0.29443, + "26": 0.28879, + "27": 0.28844, + "28": 0.28873, + "29": 0.28741, + "30": 0.28737, + "31": 0.28905, + "32": 0.28701, + "33": 0.28706, + "34": 0.28739, + "35": 0.28701, + "36": 0.28751, + "37": 0.28826, + "38": 0.28792, + "39": 0.28663, + "40": 0.28805, + "41": 0.28776, + "42": 0.28855, + "43": 0.28777, + "44": 0.28801, + "45": 0.2885, + "46": 0.28907, + "47": 0.28755, + "48": 0.28719, + "49": 0.28878, + "50": 0.28677 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json new file mode 100644 index 00000000000..00b01bf0048 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.78519, + "2": 10.81141, + "3": 10.79553, + "4": 10.78363, + "5": 10.83004, + "6": 10.8417, + "7": 10.81276, + "8": 10.81576, + "9": 10.82408, + "10": 10.75834, + "11": 10.86449, + "12": 10.83694, + "13": 10.84627, + "14": 10.85637, + "15": 10.79099, + "16": 10.78206, + "17": 10.73911, + "18": 10.77969, + "19": 10.77703, + "20": 10.69044, + "21": 10.66086, + "22": 10.51537, + "23": 10.68288, + "24": 10.54469, + "25": 10.49929, + "26": 10.5582, + "27": 10.57541, + "28": 10.54334, + "29": 10.55412, + "30": 10.31857, + "31": 10.06911, + "32": 10.42007, + "33": 10.42435, + "34": 10.16897, + "35": 10.22944, + "36": 10.19358, + "37": 10.29976, + "38": 10.1465, + "39": 10.37282, + "40": 10.04287, + "41": 10.09714, + "42": 10.17339, + "43": 9.79317, + "44": 9.91871, + "45": 9.79389, + "46": 9.77586, + "47": 10.10721, + "48": 9.81704, + "49": 9.48887, + "50": 9.87988 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 26764.0, + "2": 29274.0, + "3": 28827.0, + "4": 27434.0, + "5": 31796.0, + "6": 32589.0, + "7": 31434.0, + "8": 27373.0, + "9": 30359.0, + "10": 25775.0, + "11": 33420.0, + "12": 30835.0, + "13": 31910.0, + "14": 33275.0, + "15": 30931.0, + "16": 31713.0, + "17": 31263.0, + "18": 30574.0, + "19": 31200.0, + "20": 28330.0, + "21": 28769.0, + "22": 27643.0, + "23": 34373.0, + "24": 28585.0, + "25": 27693.0, + "26": 30668.0, + "27": 31536.0, + "28": 33620.0, + "29": 34457.0, + "30": 29810.0, + "31": 27999.0, + "32": 33871.0, + "33": 35836.0, + "34": 30770.0, + "35": 32783.0, + "36": 33452.0, + "37": 36437.0, + "38": 34114.0, + "39": 37433.0, + "40": 35063.0, + "41": 34239.0, + "42": 36041.0, + "43": 33976.0, + "44": 33303.0, + "45": 35148.0, + "46": 36112.0, + "47": 38996.0, + "48": 34779.0, + "49": 34913.0, + "50": 38547.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1364012544.0, + "2": 1364020736.0, + "3": 1364009472.0, + "4": 1364020224.0, + "5": 1364007424.0, + "6": 1364005888.0, + "7": 1364011520.0, + "8": 1364006912.0, + "9": 1364019712.0, + "10": 1364001280.0, + "11": 1364009984.0, + "12": 1364022784.0, + "13": 1364008960.0, + "14": 1364003840.0, + "15": 1364007936.0, + "16": 1363994112.0, + "17": 1363993088.0, + "18": 1364017152.0, + "19": 1364009984.0, + "20": 1363999744.0, + "21": 1364006400.0, + "22": 1363983360.0, + "23": 1363997696.0, + "24": 1364001792.0, + "25": 1363992576.0, + "26": 1364003328.0, + "27": 1363998720.0, + "28": 1363982336.0, + "29": 1363995136.0, + "30": 1363979776.0, + "31": 1363991552.0, + "32": 1363983872.0, + "33": 1363983872.0, + "34": 1363997184.0, + "35": 1363991040.0, + "36": 1363984896.0, + "37": 1363998720.0, + "38": 1363991040.0, + "39": 1363976704.0, + "40": 1363971584.0, + "41": 1363983872.0, + "42": 1363977728.0, + "43": 1363967488.0, + "44": 1363974656.0, + "45": 1363974144.0, + "46": 1363965952.0, + "47": 1363952640.0, + "48": 1363951616.0, + "49": 1363960320.0, + "50": 1363956736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3404911104.0, + "2": 3972516352.0, + "3": 3976973312.0, + "4": 3976973312.0, + "5": 3976973312.0, + "6": 3976973312.0, + "7": 3976973312.0, + "8": 3976973312.0, + "9": 3976973312.0, + "10": 3976973312.0, + "11": 3976973312.0, + "12": 3976975872.0, + "13": 3976975872.0, + "14": 3976975872.0, + "15": 3976975872.0, + "16": 3976975872.0, + "17": 3976975872.0, + "18": 3976975872.0, + "19": 3976975872.0, + "20": 3976975872.0, + "21": 3976975872.0, + "22": 3976975872.0, + "23": 3976975872.0, + "24": 3976975872.0, + "25": 3976975872.0, + "26": 3976975872.0, + "27": 3976975872.0, + "28": 3976975872.0, + "29": 3976975872.0, + "30": 3976975872.0, + "31": 3976975872.0, + "32": 3976975872.0, + "33": 3976975872.0, + "34": 3976975872.0, + "35": 3976975872.0, + "36": 3976975872.0, + "37": 3976975872.0, + "38": 3976975872.0, + "39": 3976975872.0, + "40": 3976975872.0, + "41": 3976975872.0, + "42": 3976975872.0, + "43": 3976975872.0, + "44": 3976975872.0, + "45": 3976975872.0, + "46": 3976975872.0, + "47": 3976975872.0, + "48": 3976975872.0, + "49": 3976975872.0, + "50": 3976975872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.13471, + "2": 0.38969, + "3": 0.3103, + "4": 0.30799, + "5": 0.30603, + "6": 0.30643, + "7": 0.30599, + "8": 0.30393, + "9": 0.31467, + "10": 0.306, + "11": 0.30312, + "12": 0.30329, + "13": 0.30269, + "14": 0.30471, + "15": 0.30849, + "16": 0.30377, + "17": 0.30261, + "18": 0.3009, + "19": 0.30087, + "20": 0.30146, + "21": 0.30129, + "22": 0.30052, + "23": 0.30116, + "24": 0.2999, + "25": 0.30045, + "26": 0.30179, + "27": 0.30152, + "28": 0.30154, + "29": 0.30096, + "30": 0.31152, + "31": 0.31066, + "32": 0.30418, + "33": 0.35425, + "34": 0.30235, + "35": 0.30245, + "36": 0.302, + "37": 0.31083, + "38": 0.30275, + "39": 0.30205, + "40": 0.30289, + "41": 0.30375, + "42": 0.30405, + "43": 0.30174, + "44": 0.30207, + "45": 0.303, + "46": 0.30186, + "47": 0.30147, + "48": 0.30311, + "49": 0.30205, + "50": 0.3031 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json index e640ba5b102..9114b4bb385 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.8174, "5": 10.86097, "10": 10.80463, "15": 10.79539, "20": 10.71135, "25": 10.51962, "30": 10.33346, "35": 10.22683, "40": 10.041, "45": 9.7722, "50": 9.86932}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 26535.0, "5": 31546.0, "10": 25541.0, "15": 30463.0, "20": 28331.0, "25": 27263.0, "30": 29642.0, "35": 32022.0, "40": 34911.0, "45": 35017.0, "50": 38975.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1356172288.0, "5": 1356170240.0, "10": 1356160000.0, "15": 1356180480.0, "20": 1356170240.0, "25": 1356182528.0, "30": 1356198400.0, "35": 1356182016.0, "40": 1356156416.0, "45": 1356140544.0, "50": 1356098560.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 3163797504.0, "5": 3751905792.0, "10": 3759570944.0, "15": 3759570944.0, "20": 3759570944.0, "25": 3759570944.0, "30": 3770054144.0, "35": 3770054144.0, "40": 3770054144.0, "45": 3770054144.0, "50": 3770054144.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 25.66526, "5": 0.29505, "10": 0.28776, "15": 0.28997, "20": 0.28516, "25": 0.29206, "30": 0.2932, "35": 0.29941, "40": 0.30125, "45": 0.29922, "50": 0.30133}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.78519, + "2": 10.81141, + "3": 10.79553, + "4": 10.78363, + "5": 10.83004, + "6": 10.8417, + "7": 10.81276, + "8": 10.81576, + "9": 10.82408, + "10": 10.75834, + "11": 10.86449, + "12": 10.83694, + "13": 10.84627, + "14": 10.85637, + "15": 10.79099, + "16": 10.78206, + "17": 10.73911, + "18": 10.77969, + "19": 10.77703, + "20": 10.69044, + "21": 10.66086, + "22": 10.51537, + "23": 10.68288, + "24": 10.54469, + "25": 10.49929, + "26": 10.5582, + "27": 10.57541, + "28": 10.54334, + "29": 10.55412, + "30": 10.31857, + "31": 10.06911, + "32": 10.42007, + "33": 10.42435, + "34": 10.16897, + "35": 10.22944, + "36": 10.19358, + "37": 10.29976, + "38": 10.1465, + "39": 10.37282, + "40": 10.04287, + "41": 10.09714, + "42": 10.17339, + "43": 9.79317, + "44": 9.91871, + "45": 9.79389, + "46": 9.77586, + "47": 10.10721, + "48": 9.81704, + "49": 9.48887, + "50": 9.87988 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 26764.0, + "2": 29274.0, + "3": 28827.0, + "4": 27434.0, + "5": 31796.0, + "6": 32589.0, + "7": 31434.0, + "8": 27373.0, + "9": 30359.0, + "10": 25775.0, + "11": 33420.0, + "12": 30835.0, + "13": 31910.0, + "14": 33275.0, + "15": 30931.0, + "16": 31713.0, + "17": 31263.0, + "18": 30574.0, + "19": 31200.0, + "20": 28330.0, + "21": 28769.0, + "22": 27643.0, + "23": 34373.0, + "24": 28585.0, + "25": 27693.0, + "26": 30668.0, + "27": 31536.0, + "28": 33620.0, + "29": 34457.0, + "30": 29810.0, + "31": 27999.0, + "32": 33871.0, + "33": 35836.0, + "34": 30770.0, + "35": 32783.0, + "36": 33452.0, + "37": 36437.0, + "38": 34114.0, + "39": 37433.0, + "40": 35063.0, + "41": 34239.0, + "42": 36041.0, + "43": 33976.0, + "44": 33303.0, + "45": 35148.0, + "46": 36112.0, + "47": 38996.0, + "48": 34779.0, + "49": 34913.0, + "50": 38547.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1364012544.0, + "2": 1364020736.0, + "3": 1364009472.0, + "4": 1364020224.0, + "5": 1364007424.0, + "6": 1364005888.0, + "7": 1364011520.0, + "8": 1364006912.0, + "9": 1364019712.0, + "10": 1364001280.0, + "11": 1364009984.0, + "12": 1364022784.0, + "13": 1364008960.0, + "14": 1364003840.0, + "15": 1364007936.0, + "16": 1363994112.0, + "17": 1363993088.0, + "18": 1364017152.0, + "19": 1364009984.0, + "20": 1363999744.0, + "21": 1364006400.0, + "22": 1363983360.0, + "23": 1363997696.0, + "24": 1364001792.0, + "25": 1363992576.0, + "26": 1364003328.0, + "27": 1363998720.0, + "28": 1363982336.0, + "29": 1363995136.0, + "30": 1363979776.0, + "31": 1363991552.0, + "32": 1363983872.0, + "33": 1363983872.0, + "34": 1363997184.0, + "35": 1363991040.0, + "36": 1363984896.0, + "37": 1363998720.0, + "38": 1363991040.0, + "39": 1363976704.0, + "40": 1363971584.0, + "41": 1363983872.0, + "42": 1363977728.0, + "43": 1363967488.0, + "44": 1363974656.0, + "45": 1363974144.0, + "46": 1363965952.0, + "47": 1363952640.0, + "48": 1363951616.0, + "49": 1363960320.0, + "50": 1363956736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3404911104.0, + "2": 3972516352.0, + "3": 3976973312.0, + "4": 3976973312.0, + "5": 3976973312.0, + "6": 3976973312.0, + "7": 3976973312.0, + "8": 3976973312.0, + "9": 3976973312.0, + "10": 3976973312.0, + "11": 3976973312.0, + "12": 3976975872.0, + "13": 3976975872.0, + "14": 3976975872.0, + "15": 3976975872.0, + "16": 3976975872.0, + "17": 3976975872.0, + "18": 3976975872.0, + "19": 3976975872.0, + "20": 3976975872.0, + "21": 3976975872.0, + "22": 3976975872.0, + "23": 3976975872.0, + "24": 3976975872.0, + "25": 3976975872.0, + "26": 3976975872.0, + "27": 3976975872.0, + "28": 3976975872.0, + "29": 3976975872.0, + "30": 3976975872.0, + "31": 3976975872.0, + "32": 3976975872.0, + "33": 3976975872.0, + "34": 3976975872.0, + "35": 3976975872.0, + "36": 3976975872.0, + "37": 3976975872.0, + "38": 3976975872.0, + "39": 3976975872.0, + "40": 3976975872.0, + "41": 3976975872.0, + "42": 3976975872.0, + "43": 3976975872.0, + "44": 3976975872.0, + "45": 3976975872.0, + "46": 3976975872.0, + "47": 3976975872.0, + "48": 3976975872.0, + "49": 3976975872.0, + "50": 3976975872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.62035, + "2": 0.36752, + "3": 0.30562, + "4": 0.29876, + "5": 0.298, + "6": 0.29743, + "7": 0.29729, + "8": 0.2967, + "9": 0.29751, + "10": 0.29912, + "11": 0.29575, + "12": 0.29589, + "13": 0.29696, + "14": 0.29898, + "15": 0.30053, + "16": 0.30093, + "17": 0.2977, + "18": 0.2973, + "19": 0.29596, + "20": 0.29757, + "21": 0.2967, + "22": 0.29963, + "23": 0.29707, + "24": 0.29748, + "25": 0.29701, + "26": 0.29838, + "27": 0.29889, + "28": 0.29962, + "29": 0.30399, + "30": 0.30932, + "31": 0.30553, + "32": 0.29765, + "33": 0.30499, + "34": 0.29754, + "35": 0.29747, + "36": 0.29801, + "37": 0.30768, + "38": 0.29693, + "39": 0.29912, + "40": 0.299, + "41": 0.2982, + "42": 0.37256, + "43": 0.29865, + "44": 0.29774, + "45": 0.29961, + "46": 0.2988, + "47": 0.30454, + "48": 0.30466, + "49": 0.30093, + "50": 0.29883 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json new file mode 100644 index 00000000000..390f4b8f13f --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.77518, + "2": 10.78038, + "3": 10.79302, + "4": 10.74107, + "5": 10.82013, + "6": 10.82951, + "7": 10.7953, + "8": 10.78263, + "9": 10.79278, + "10": 10.7446, + "11": 10.85147, + "12": 10.82613, + "13": 10.82825, + "14": 10.85504, + "15": 10.75536, + "16": 10.75777, + "17": 10.72319, + "18": 10.76274, + "19": 10.75075, + "20": 10.66587, + "21": 10.6419, + "22": 10.47523, + "23": 10.66959, + "24": 10.54157, + "25": 10.4825, + "26": 10.55255, + "27": 10.57459, + "28": 10.55159, + "29": 10.5668, + "30": 10.31134, + "31": 10.01921, + "32": 10.42655, + "33": 10.42294, + "34": 10.14739, + "35": 10.21574, + "36": 10.15811, + "37": 10.30279, + "38": 10.14031, + "39": 10.36301, + "40": 10.02669, + "41": 10.07635, + "42": 10.16156, + "43": 9.74374, + "44": 9.88962, + "45": 9.75874, + "46": 9.73618, + "47": 10.0844, + "48": 9.78532, + "49": 9.45072, + "50": 9.85634 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 27105.0, + "2": 28791.0, + "3": 29282.0, + "4": 27583.0, + "5": 31595.0, + "6": 32831.0, + "7": 31023.0, + "8": 27107.0, + "9": 30780.0, + "10": 25505.0, + "11": 33684.0, + "12": 30235.0, + "13": 32960.0, + "14": 32880.0, + "15": 30405.0, + "16": 32455.0, + "17": 30933.0, + "18": 30623.0, + "19": 30803.0, + "20": 28593.0, + "21": 29002.0, + "22": 27030.0, + "23": 34463.0, + "24": 29154.0, + "25": 27827.0, + "26": 31119.0, + "27": 32108.0, + "28": 33412.0, + "29": 34737.0, + "30": 30465.0, + "31": 28775.0, + "32": 33115.0, + "33": 34745.0, + "34": 30785.0, + "35": 32116.0, + "36": 33968.0, + "37": 36757.0, + "38": 34150.0, + "39": 37240.0, + "40": 35353.0, + "41": 34638.0, + "42": 36703.0, + "43": 34601.0, + "44": 33783.0, + "45": 35388.0, + "46": 35484.0, + "47": 40591.0, + "48": 36671.0, + "49": 36174.0, + "50": 38231.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1566715392.0, + "2": 1564834816.0, + "3": 1564860416.0, + "4": 1564313088.0, + "5": 1564120576.0, + "6": 1565958144.0, + "7": 1564614656.0, + "8": 1563520512.0, + "9": 1563819008.0, + "10": 1563891712.0, + "11": 1563430400.0, + "12": 1564003840.0, + "13": 1564584960.0, + "14": 1566458880.0, + "15": 1564517376.0, + "16": 1565116928.0, + "17": 1564445184.0, + "18": 1566232576.0, + "19": 1565403136.0, + "20": 1565009408.0, + "21": 1568513024.0, + "22": 1564807168.0, + "23": 1567177216.0, + "24": 1564333568.0, + "25": 1566986240.0, + "26": 1565763584.0, + "27": 1563955200.0, + "28": 1564926464.0, + "29": 1563690496.0, + "30": 1563930624.0, + "31": 1564862464.0, + "32": 1564173824.0, + "33": 1565158400.0, + "34": 1564895232.0, + "35": 1564069376.0, + "36": 1565074944.0, + "37": 1563990016.0, + "38": 1565044736.0, + "39": 1563646976.0, + "40": 1565059072.0, + "41": 1566659584.0, + "42": 1564623360.0, + "43": 1563140608.0, + "44": 1565100032.0, + "45": 1567626752.0, + "46": 1565074944.0, + "47": 1564590080.0, + "48": 1564522496.0, + "49": 1563403776.0, + "50": 1564356096.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3737165824.0, + "2": 4315653632.0, + "3": 4318730752.0, + "4": 4318730752.0, + "5": 4318730752.0, + "6": 4346984960.0, + "7": 4346984960.0, + "8": 4346984960.0, + "9": 4346984960.0, + "10": 4346984960.0, + "11": 4346984960.0, + "12": 4346984960.0, + "13": 4346984960.0, + "14": 4346984960.0, + "15": 4346984960.0, + "16": 4346984960.0, + "17": 4346984960.0, + "18": 4346984960.0, + "19": 4346984960.0, + "20": 4346984960.0, + "21": 4346984960.0, + "22": 4346984960.0, + "23": 4346984960.0, + "24": 4346984960.0, + "25": 4346984960.0, + "26": 4346984960.0, + "27": 4346984960.0, + "28": 4346984960.0, + "29": 4346984960.0, + "30": 4346984960.0, + "31": 4346984960.0, + "32": 4346984960.0, + "33": 4346984960.0, + "34": 4346984960.0, + "35": 4346984960.0, + "36": 4346984960.0, + "37": 4346984960.0, + "38": 4346984960.0, + "39": 4346984960.0, + "40": 4346984960.0, + "41": 4346984960.0, + "42": 4346984960.0, + "43": 4346984960.0, + "44": 4346984960.0, + "45": 4346984960.0, + "46": 4346984960.0, + "47": 4346984960.0, + "48": 4346984960.0, + "49": 4346984960.0, + "50": 4346984960.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 19.53588, + "2": 0.42103, + "3": 0.37171, + "4": 0.32787, + "5": 0.32504, + "6": 0.29089, + "7": 0.29215, + "8": 0.28165, + "9": 0.287, + "10": 0.28805, + "11": 0.27222, + "12": 0.2768, + "13": 0.26942, + "14": 0.27369, + "15": 0.27287, + "16": 0.26492, + "17": 0.27231, + "18": 0.26408, + "19": 0.26251, + "20": 0.26769, + "21": 0.26539, + "22": 0.27591, + "23": 0.26191, + "24": 0.27512, + "25": 0.26855, + "26": 0.27323, + "27": 0.26526, + "28": 0.26748, + "29": 0.27054, + "30": 0.28834, + "31": 0.27784, + "32": 0.27006, + "33": 0.29268, + "34": 0.27918, + "35": 0.29002, + "36": 0.28672, + "37": 0.28354, + "38": 0.27713, + "39": 0.28145, + "40": 0.62986, + "41": 0.2715, + "42": 0.2633, + "43": 0.27508, + "44": 0.28176, + "45": 0.26745, + "46": 0.27357, + "47": 0.26873, + "48": 0.26907, + "49": 0.26843, + "50": 0.63541 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..779e572fdd4 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.59918, + "2": 0.61895, + "3": 0.56068, + "4": 0.55629, + "5": 0.55614, + "6": 0.55955, + "7": 0.55763, + "8": 0.55984, + "9": 0.55, + "10": 0.5499, + "11": 0.55012, + "12": 0.54844, + "13": 0.55795, + "14": 0.54851, + "15": 0.5504, + "16": 0.55068, + "17": 0.54716, + "18": 0.55003, + "19": 0.54872, + "20": 0.54841, + "21": 0.55411, + "22": 0.55114, + "23": 0.54838, + "24": 0.54877, + "25": 0.54811, + "26": 0.54808, + "27": 0.54959, + "28": 0.55486, + "29": 0.55193, + "30": 0.54966, + "31": 0.54964, + "32": 0.55053, + "33": 0.55025, + "34": 0.55174, + "35": 0.55, + "36": 0.55117, + "37": 0.55004, + "38": 0.55089, + "39": 0.54857, + "40": 0.55402, + "41": 0.55168, + "42": 0.54916, + "43": 0.55132, + "44": 0.54941, + "45": 0.54876, + "46": 0.55724, + "47": 0.54849, + "48": 0.55007, + "49": 0.55215, + "50": 0.54754 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json index 7aae0170419..a186febffbe 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.79288, "5": 10.828, "10": 10.76831, "15": 10.79073, "20": 10.7066, "25": 10.54061, "30": 10.39485, "35": 10.30488, "40": 10.13079, "45": 9.90236, "50": 9.97938}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 5601.0, "5": 6601.0, "10": 5254.0, "15": 6250.0, "20": 5969.0, "25": 5816.0, "30": 6181.0, "35": 6751.0, "40": 6831.0, "45": 6881.0, "50": 7684.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 458211840.0, "5": 458212864.0, "10": 458214912.0, "15": 458215424.0, "20": 458212864.0, "25": 458215424.0, "30": 458211840.0, "35": 458214400.0, "40": 458214912.0, "45": 458213888.0, "50": 458214912.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1016394240.0, "5": 1180904960.0, "10": 1180934144.0, "15": 1180990976.0, "20": 1180990976.0, "25": 1181222912.0, "30": 1181222912.0, "35": 1181468160.0, "40": 1181468160.0, "45": 1183467008.0, "50": 1183467008.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 16.31521, "5": 0.56321, "10": 0.56536, "15": 0.56469, "20": 0.55362, "25": 0.55331, "30": 0.55482, "35": 0.57032, "40": 0.56489, "45": 0.55226, "50": 0.54242}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.49228, + "2": 0.63481, + "3": 0.56951, + "4": 0.57807, + "5": 0.581, + "6": 0.58159, + "7": 0.5705, + "8": 0.56929, + "9": 0.56794, + "10": 0.56314, + "11": 0.57935, + "12": 0.57294, + "13": 0.56865, + "14": 0.56698, + "15": 0.56505, + "16": 0.56266, + "17": 0.56337, + "18": 0.56237, + "19": 0.56197, + "20": 0.5626, + "21": 0.5642, + "22": 0.56373, + "23": 0.57291, + "24": 0.56432, + "25": 0.56287, + "26": 0.56295, + "27": 0.56146, + "28": 0.56459, + "29": 0.56415, + "30": 0.56587, + "31": 0.5671, + "32": 0.56896, + "33": 0.57526, + "34": 0.57281, + "35": 0.57407, + "36": 0.57321, + "37": 0.57403, + "38": 0.57296, + "39": 0.57248, + "40": 0.57089, + "41": 0.57201, + "42": 0.5661, + "43": 0.57044, + "44": 0.56777, + "45": 0.56877, + "46": 0.57143, + "47": 0.57031, + "48": 0.56952, + "49": 0.57353, + "50": 0.56636 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json new file mode 100644 index 00000000000..63611a809be --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.94206, + "2": 0.63285, + "3": 0.57182, + "4": 0.57773, + "5": 0.57872, + "6": 0.57643, + "7": 0.57659, + "8": 0.57355, + "9": 0.57629, + "10": 0.56867, + "11": 0.56833, + "12": 0.56771, + "13": 0.56904, + "14": 0.59894, + "15": 0.67038, + "16": 0.5641, + "17": 0.56097, + "18": 0.56495, + "19": 0.56059, + "20": 0.56119, + "21": 0.55935, + "22": 0.55364, + "23": 0.55179, + "24": 0.55091, + "25": 0.55193, + "26": 0.55148, + "27": 0.55193, + "28": 0.55181, + "29": 0.55146, + "30": 0.55214, + "31": 0.55292, + "32": 0.55164, + "33": 0.55141, + "34": 0.54974, + "35": 0.55215, + "36": 0.55139, + "37": 0.55509, + "38": 0.5519, + "39": 0.55134, + "40": 0.55133, + "41": 0.55143, + "42": 0.5495, + "43": 0.54991, + "44": 0.5491, + "45": 0.54833, + "46": 0.54896, + "47": 0.54857, + "48": 0.5489, + "49": 0.54919, + "50": 0.54936 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json index 5f0c72de4dd..4ebfff8da76 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.79288, "5": 10.828, "10": 10.76831, "15": 10.79073, "20": 10.7066, "25": 10.54061, "30": 10.39485, "35": 10.30488, "40": 10.13079, "45": 9.90236, "50": 9.97938}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 5601.0, "5": 6601.0, "10": 5254.0, "15": 6250.0, "20": 5969.0, "25": 5816.0, "30": 6181.0, "35": 6751.0, "40": 6831.0, "45": 6881.0, "50": 7684.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 458211840.0, "5": 458212864.0, "10": 458214912.0, "15": 458215424.0, "20": 458212864.0, "25": 458215424.0, "30": 458211840.0, "35": 458214400.0, "40": 458214912.0, "45": 458213888.0, "50": 458214912.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1016394240.0, "5": 1180904960.0, "10": 1180934144.0, "15": 1180990976.0, "20": 1180990976.0, "25": 1181222912.0, "30": 1181222912.0, "35": 1181468160.0, "40": 1181468160.0, "45": 1183467008.0, "50": 1183467008.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 19.9711, "5": 0.5845, "10": 0.59728, "15": 0.60305, "20": 0.58013, "25": 0.57491, "30": 0.5625, "35": 0.56368, "40": 0.56421, "45": 0.57011, "50": 0.58956}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.27777, + "2": 0.60806, + "3": 0.55409, + "4": 0.55324, + "5": 0.54815, + "6": 0.54698, + "7": 0.54712, + "8": 0.55008, + "9": 0.55718, + "10": 0.55527, + "11": 0.55082, + "12": 0.56208, + "13": 0.55625, + "14": 0.55717, + "15": 0.56582, + "16": 0.55953, + "17": 0.57188, + "18": 0.55508, + "19": 0.55956, + "20": 0.55934, + "21": 0.55676, + "22": 0.55842, + "23": 0.55867, + "24": 0.55987, + "25": 0.55941, + "26": 0.55642, + "27": 0.55364, + "28": 0.55209, + "29": 0.55397, + "30": 0.55602, + "31": 0.55344, + "32": 0.55195, + "33": 0.56308, + "34": 0.55588, + "35": 0.55251, + "36": 0.55314, + "37": 0.55563, + "38": 0.56708, + "39": 0.5661, + "40": 0.56725, + "41": 0.5663, + "42": 0.56565, + "43": 0.5725, + "44": 0.56736, + "45": 0.5674, + "46": 0.56751, + "47": 0.56642, + "48": 0.56257, + "49": 0.56841, + "50": 0.56452 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json new file mode 100644 index 00000000000..2c1e24efd92 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.85676, + "2": 0.38679, + "3": 0.33955, + "4": 0.34149, + "5": 0.33738, + "6": 0.3367, + "7": 0.33549, + "8": 0.33607, + "9": 0.33582, + "10": 0.33626, + "11": 0.33555, + "12": 0.33476, + "13": 0.33518, + "14": 0.33155, + "15": 0.33236, + "16": 0.33276, + "17": 0.33237, + "18": 0.33234, + "19": 0.33191, + "20": 0.33586, + "21": 0.334, + "22": 0.33303, + "23": 0.33299, + "24": 0.33339, + "25": 0.33301, + "26": 0.33368, + "27": 0.33213, + "28": 0.33242, + "29": 0.33251, + "30": 0.33285, + "31": 0.33233, + "32": 0.33175, + "33": 0.33133, + "34": 0.33224, + "35": 0.3317, + "36": 0.33247, + "37": 0.33189, + "38": 0.3324, + "39": 0.33157, + "40": 0.33224, + "41": 0.33218, + "42": 0.33335, + "43": 0.33381, + "44": 0.33284, + "45": 0.33399, + "46": 0.33341, + "47": 0.33248, + "48": 0.33301, + "49": 0.33248, + "50": 0.33145 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..c1d418bd4a8 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.62289, + "2": 0.39639, + "3": 0.35696, + "4": 0.3674, + "5": 0.3577, + "6": 0.35509, + "7": 0.35808, + "8": 0.35462, + "9": 0.35492, + "10": 0.35391, + "11": 0.35546, + "12": 0.35515, + "13": 0.35432, + "14": 0.35491, + "15": 0.35335, + "16": 0.35396, + "17": 0.35304, + "18": 0.35201, + "19": 0.35198, + "20": 0.35194, + "21": 0.35412, + "22": 0.35208, + "23": 0.3541, + "24": 0.3519, + "25": 0.38526, + "26": 0.39355, + "27": 0.39377, + "28": 0.3934, + "29": 0.39406, + "30": 0.39393, + "31": 0.38905, + "32": 0.3893, + "33": 0.39052, + "34": 0.39004, + "35": 0.38977, + "36": 0.39027, + "37": 0.38961, + "38": 0.38929, + "39": 0.38944, + "40": 0.38968, + "41": 0.38938, + "42": 0.38964, + "43": 0.3909, + "44": 0.39131, + "45": 0.38436, + "46": 0.34892, + "47": 0.34845, + "48": 0.34853, + "49": 0.3474, + "50": 0.34681 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json index 1f335e82393..478bae6fdec 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts_dgx_a100.json @@ -1 +1,287 @@ -{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.82416, "5": 10.84149, "10": 10.79041, "15": 10.8302, "20": 10.73076, "25": 10.57969, "30": 10.41029, "35": 10.31774, "40": 10.14481, "45": 9.91307, "50": 9.97569}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 12709.0, "5": 15699.0, "10": 12503.0, "15": 14756.0, "20": 13658.0, "25": 13295.0, "30": 14801.0, "35": 15233.0, "40": 16737.0, "45": 16130.0, "50": 19242.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 625788416.0, "5": 625793536.0, "10": 625788928.0, "15": 625790976.0, "20": 625793024.0, "25": 625789440.0, "30": 625790976.0, "35": 625789952.0, "40": 625793536.0, "45": 625790976.0, "50": 625793024.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1818507264.0, "5": 2049478144.0, "10": 2049478144.0, "15": 2050050048.0, "20": 2050050048.0, "25": 2050050048.0, "30": 2050050048.0, "35": 2050050048.0, "40": 2050050048.0, "45": 2050050048.0, "50": 2050050048.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 16.64702, "5": 0.34446, "10": 0.34582, "15": 0.34645, "20": 0.34853, "25": 0.34852, "30": 0.34703, "35": 0.34516, "40": 0.34201, "45": 0.34861, "50": 0.3491}}} \ No newline at end of file +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.46534, + "2": 0.4102, + "3": 0.34219, + "4": 0.34093, + "5": 0.34255, + "6": 0.33706, + "7": 0.33661, + "8": 0.33616, + "9": 0.33565, + "10": 0.33568, + "11": 0.33538, + "12": 0.33457, + "13": 0.33488, + "14": 0.33416, + "15": 0.33683, + "16": 0.33684, + "17": 0.33708, + "18": 0.33815, + "19": 0.33805, + "20": 0.33696, + "21": 0.33675, + "22": 0.33623, + "23": 0.33752, + "24": 0.33699, + "25": 0.3409, + "26": 0.33513, + "27": 0.33524, + "28": 0.33491, + "29": 0.33714, + "30": 0.33571, + "31": 0.33638, + "32": 0.33629, + "33": 0.3369, + "34": 0.33685, + "35": 0.33651, + "36": 0.33539, + "37": 0.33561, + "38": 0.33636, + "39": 0.33558, + "40": 0.3356, + "41": 0.33618, + "42": 0.33669, + "43": 0.33535, + "44": 0.3362, + "45": 0.3354, + "46": 0.33686, + "47": 0.33486, + "48": 0.33657, + "49": 0.33563, + "50": 0.33513 + } + } +} \ No newline at end of file diff --git a/tests/test_utils/python_scripts/download_golden_values.py b/tests/test_utils/python_scripts/download_golden_values.py index 650867f231f..e2294b32fbb 100644 --- a/tests/test_utils/python_scripts/download_golden_values.py +++ b/tests/test_utils/python_scripts/download_golden_values.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import logging import os import pathlib @@ -91,8 +93,6 @@ def main(pipeline_id: int, only_failing: bool): continue golden_values_source_name = golden_values_source.name - golden_values_source_name = golden_values_source_name.replace("_dgx_h100", "") - golden_values_source_name = golden_values_source_name.replace("_dgx_a100", "") golden_values_source_name = golden_values_source_name.replace( "generations", "golden_values" ) diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py index 7dc4a7357a7..846ca8a1521 100644 --- a/tests/test_utils/python_scripts/launch_jet_workload.py +++ b/tests/test_utils/python_scripts/launch_jet_workload.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import json import logging import os @@ -289,6 +291,7 @@ def is_flaky_failure(concat_allranks_logs: str) -> bool: or "unspecified launch failure" in concat_allranks_logs or "free(): corrupted unsorted chunks" in concat_allranks_logs or "Segfault encountered" in concat_allranks_logs + or "Fatal glibc error" in concat_allranks_logs ) diff --git a/uv.lock b/uv.lock index 5451e635d0c..3de5e29b62b 100644 --- a/uv.lock +++ b/uv.lock @@ -300,12 +300,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] -[[package]] -name = "asciitree" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", size = 3951, upload-time = "2016-09-05T19:10:42.681Z" } - [[package]] name = "astroid" version = "3.2.4" @@ -986,82 +980,6 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] -[[package]] -name = "crc32c" -version = "2.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/a0/28b4686a8db0bb0f77970f4c6ccede90d1d5740a1d4b4703bd54c3e75655/crc32c-2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2c0f4eb01fe7c0a3e3f973a418e04d52101bb077dd77626fd80c658ec60aaf95", size = 66321, upload-time = "2025-10-17T06:18:53.543Z" }, - { url = "https://files.pythonhosted.org/packages/76/1f/1697f5b8b770f715ed9b264d79e36b4f77ae0527f81f3c749ef08937a32e/crc32c-2.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6baefcfbca82b1a9678455416da24f18629769a76920c640d5a538620a7d12bb", size = 62985, upload-time = "2025-10-17T06:18:54.97Z" }, - { url = "https://files.pythonhosted.org/packages/e0/e5/333cfa5ffa8d5779733aced2b984b5e5139b4a8ceaa2c6bc563e9a1092f3/crc32c-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7f959fcf6c5aad1c4a653ee1a50f05760dab1d1c35d98ec4d7f0f68643f7612", size = 61517, upload-time = "2025-10-17T06:18:55.795Z" }, - { url = "https://files.pythonhosted.org/packages/e1/d8/362a009e8140dd926a153b44d56753e3aa7cb50aca243779a84adadbff11/crc32c-2.8-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bb678507a4e4cf3f0506607b046ecc4ed1c58a19e08a3fb3c2d25441c480bf1", size = 79385, upload-time = "2025-10-17T06:18:56.598Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0d4ea3aa71ffb15f1285669d23024cc40779388ce32157d339dc2584491c/crc32c-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a16f7ffa4c242a909558565567cbba95148603717b53538ea299c98da68e7a9", size = 80965, upload-time = "2025-10-17T06:18:57.384Z" }, - { url = "https://files.pythonhosted.org/packages/20/44/d77657aaca4a2c0283f2356a3da6f8e91b003567bb8f09daaf540cbf192f/crc32c-2.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0184369aad562d801f91f454c81f56b9ecb966f6b96684c4d6cf82fc8741d2ad", size = 79993, upload-time = "2025-10-17T06:18:58.503Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c0/07017a93ebf85d9408028b7e03ef96d5c6bfb14cb77cfe90d35eedcc1501/crc32c-2.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86d2eeb5f0189bd803720abe7387019328ea34c4acde62999e5723f789bc316b", size = 79243, upload-time = "2025-10-17T06:18:59.273Z" }, - { url = "https://files.pythonhosted.org/packages/c7/1a/b3c5ac4cf2fd1f82395173d0bd8e1a15d09f0bc1eccdf10ea7f8caaccd67/crc32c-2.8-cp310-cp310-win32.whl", hash = "sha256:51da61904a9e753780a2e6011885677d601db1fa840be4b68799643a113e6f08", size = 64888, upload-time = "2025-10-17T06:19:00.089Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f2/60c45fc7bb2221d3c93c7a872e921be591f40d45228fe46f879b1d8c0424/crc32c-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:b2d6a1f2500daaf2e4b08f97ad0349aa2eff5faaaa5fd3350314a26eade334cd", size = 66639, upload-time = "2025-10-17T06:19:00.974Z" }, - { url = "https://files.pythonhosted.org/packages/dc/0b/5e03b22d913698e9cc563f39b9f6bbd508606bf6b8e9122cd6bf196b87ea/crc32c-2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e560a97fbb96c9897cb1d9b5076ef12fc12e2e25622530a1afd0de4240f17e1f", size = 66329, upload-time = "2025-10-17T06:19:01.771Z" }, - { url = "https://files.pythonhosted.org/packages/6b/38/2fe0051ffe8c6a650c8b1ac0da31b8802d1dbe5fa40a84e4b6b6f5583db5/crc32c-2.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6762d276d90331a490ef7e71ffee53b9c0eb053bd75a272d786f3b08d3fe3671", size = 62988, upload-time = "2025-10-17T06:19:02.953Z" }, - { url = "https://files.pythonhosted.org/packages/3e/30/5837a71c014be83aba1469c58820d287fc836512a0cad6b8fdd43868accd/crc32c-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:60670569f5ede91e39f48fb0cb4060e05b8d8704dd9e17ede930bf441b2f73ef", size = 61522, upload-time = "2025-10-17T06:19:03.796Z" }, - { url = "https://files.pythonhosted.org/packages/ca/29/63972fc1452778e2092ae998c50cbfc2fc93e3fa9798a0278650cd6169c5/crc32c-2.8-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:711743da6ccc70b3c6718c328947b0b6f34a1fe6a6c27cc6c1d69cc226bf70e9", size = 80200, upload-time = "2025-10-17T06:19:04.617Z" }, - { url = "https://files.pythonhosted.org/packages/cb/3a/60eb49d7bdada4122b3ffd45b0df54bdc1b8dd092cda4b069a287bdfcff4/crc32c-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eb4094a2054774f13b26f21bf56792bb44fa1fcee6c6ad099387a43ffbfb4fa", size = 81757, upload-time = "2025-10-17T06:19:05.496Z" }, - { url = "https://files.pythonhosted.org/packages/f5/63/6efc1b64429ef7d23bd58b75b7ac24d15df327e3ebbe9c247a0f7b1c2ed1/crc32c-2.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fff15bf2bd3e95780516baae935ed12be88deaa5ebe6143c53eb0d26a7bdc7b7", size = 80830, upload-time = "2025-10-17T06:19:06.621Z" }, - { url = "https://files.pythonhosted.org/packages/e1/eb/0ae9f436f8004f1c88f7429e659a7218a3879bd11a6b18ed1257aad7e98b/crc32c-2.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c0e11e3826668121fa53e0745635baf5e4f0ded437e8ff63ea56f38fc4f970a", size = 80095, upload-time = "2025-10-17T06:19:07.381Z" }, - { url = "https://files.pythonhosted.org/packages/9e/81/4afc9d468977a4cd94a2eb62908553345009a7c0d30e74463a15d4b48ec3/crc32c-2.8-cp311-cp311-win32.whl", hash = "sha256:38f915336715d1f1353ab07d7d786f8a789b119e273aea106ba55355dfc9101d", size = 64886, upload-time = "2025-10-17T06:19:08.497Z" }, - { url = "https://files.pythonhosted.org/packages/d6/e8/94e839c9f7e767bf8479046a207afd440a08f5c59b52586e1af5e64fa4a0/crc32c-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:60e0a765b1caab8d31b2ea80840639253906a9351d4b861551c8c8625ea20f86", size = 66639, upload-time = "2025-10-17T06:19:09.338Z" }, - { url = "https://files.pythonhosted.org/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab", size = 66379, upload-time = "2025-10-17T06:19:10.115Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68", size = 63042, upload-time = "2025-10-17T06:19:10.946Z" }, - { url = "https://files.pythonhosted.org/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2", size = 61528, upload-time = "2025-10-17T06:19:11.768Z" }, - { url = "https://files.pythonhosted.org/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2", size = 80028, upload-time = "2025-10-17T06:19:12.554Z" }, - { url = "https://files.pythonhosted.org/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1", size = 81531, upload-time = "2025-10-17T06:19:13.406Z" }, - { url = "https://files.pythonhosted.org/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e", size = 80608, upload-time = "2025-10-17T06:19:14.223Z" }, - { url = "https://files.pythonhosted.org/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c", size = 79886, upload-time = "2025-10-17T06:19:15.345Z" }, - { url = "https://files.pythonhosted.org/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76", size = 64896, upload-time = "2025-10-17T06:19:16.457Z" }, - { url = "https://files.pythonhosted.org/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2", size = 66645, upload-time = "2025-10-17T06:19:17.235Z" }, - { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, - { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, - { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, - { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, - { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, - { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, - { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, - { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, - { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, - { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, - { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, - { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, - { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, - { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, - { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, - { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, - { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, - { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, - { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, - { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, - { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, - { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, - { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, - { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, - { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, - { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, - { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, - { url = "https://files.pythonhosted.org/packages/a7/1d/dd926c68eb8aac8b142a1a10b8eb62d95212c1cf81775644373fe7cceac2/crc32c-2.8-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5833f4071da7ea182c514ba17d1eee8aec3c5be927d798222fbfbbd0f5eea02c", size = 62345, upload-time = "2025-10-17T06:20:09.39Z" }, - { url = "https://files.pythonhosted.org/packages/51/be/803404e5abea2ef2c15042edca04bbb7f625044cca879e47f186b43887c2/crc32c-2.8-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1dc4da036126ac07b39dd9d03e93e585ec615a2ad28ff12757aef7de175295a8", size = 61229, upload-time = "2025-10-17T06:20:10.236Z" }, - { url = "https://files.pythonhosted.org/packages/fc/3a/00cc578cd27ed0b22c9be25cef2c24539d92df9fa80ebd67a3fc5419724c/crc32c-2.8-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:15905fa78344654e241371c47e6ed2411f9eeb2b8095311c68c88eccf541e8b4", size = 64108, upload-time = "2025-10-17T06:20:11.072Z" }, - { url = "https://files.pythonhosted.org/packages/6b/bc/0587ef99a1c7629f95dd0c9d4f3d894de383a0df85831eb16c48a6afdae4/crc32c-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c596f918688821f796434e89b431b1698396c38bf0b56de873621528fe3ecb1e", size = 64815, upload-time = "2025-10-17T06:20:11.919Z" }, - { url = "https://files.pythonhosted.org/packages/73/42/94f2b8b92eae9064fcfb8deef2b971514065bd606231f8857ff8ae02bebd/crc32c-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8d23c4fe01b3844cb6e091044bc1cebdef7d16472e058ce12d9fadf10d2614af", size = 66659, upload-time = "2025-10-17T06:20:12.766Z" }, -] - [[package]] name = "cryptography" version = "42.0.8" @@ -1182,8 +1100,8 @@ name = "deprecated" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ @@ -1231,18 +1149,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "ebmlite" version = "3.4.1" @@ -1303,15 +1209,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/cc/1c33d05f62c9349bb80dfe789cc9a7409bdfb337a63fa347fd651d25294a/fastapi-0.120.2-py3-none-any.whl", hash = "sha256:bedcf2c14240e43d56cb9a339b32bcf15104fe6b5897c0222603cb7ec416c8eb", size = 108383, upload-time = "2025-10-29T13:47:32.978Z" }, ] -[[package]] -name = "fasteners" -version = "0.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz", hash = "sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", size = 25087, upload-time = "2025-08-11T10:19:37.785Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl", hash = "sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", size = 18702, upload-time = "2025-08-11T10:19:35.716Z" }, -] - [[package]] name = "fiddle" version = "0.3.0" @@ -1814,7 +1711,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/08/c1395a292bb23fd03bdf572a1357c5a733d3eecbab877641ceacab23db6e/importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580", size = 55767, upload-time = "2025-01-20T22:21:30.429Z" } wheels = [ @@ -1826,35 +1723,23 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "zipp", marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "zipp", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -2280,16 +2165,23 @@ dev = [ { name = "wget" }, ] lts = [ + { name = "av" }, + { name = "causal-conv1d" }, { name = "einops" }, + { name = "flashinfer-python" }, + { name = "mamba-ssm" }, + { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-lts'" }, + { name = "multi-storage-client" }, + { name = "nv-grouped-gemm" }, { name = "nvtx" }, + { name = "onnxscript", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnxscript", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-api" }, { name = "setuptools" }, { name = "tensorstore", version = "0.1.74", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tqdm" }, - { name = "transformers" }, { name = "wget" }, - { name = "zarr", version = "2.18.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "zarr", version = "3.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] mlm = [ { name = "flask-restful" }, @@ -2347,46 +2239,53 @@ test = [ { name = "pytest-random-order" }, { name = "pyyaml" }, { name = "tensorboard" }, - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] [package.metadata] requires-dist = [ { name = "av", marker = "extra == 'dev'", specifier = "<16.0.0" }, + { name = "av", marker = "extra == 'lts'", specifier = "<16.0.0" }, { name = "causal-conv1d", marker = "extra == 'dev'", specifier = "~=1.5" }, + { name = "causal-conv1d", marker = "extra == 'lts'", specifier = "~=1.5" }, { name = "einops", marker = "extra == 'dev'", specifier = "~=0.8" }, - { name = "einops", marker = "extra == 'lts'" }, + { name = "einops", marker = "extra == 'lts'", specifier = "~=0.8" }, { name = "flashinfer-python", marker = "extra == 'dev'" }, + { name = "flashinfer-python", marker = "extra == 'lts'" }, { name = "flask-restful", marker = "extra == 'mlm'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, + { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, + { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'lts'", specifier = "~=6.0" }, { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, + { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy", specifier = "<2.0.0" }, { name = "nv-grouped-gemm", marker = "extra == 'dev'", specifier = "~=1.1" }, + { name = "nv-grouped-gemm", marker = "extra == 'lts'", specifier = "~=1.1" }, { name = "nvidia-modelopt", extras = ["torch"], marker = "sys_platform != 'darwin' and extra == 'dev'" }, - { name = "nvidia-resiliency-ext", marker = "extra == 'dev'", specifier = ">=0.4.0a0,<0.5.0" }, + { name = "nvidia-resiliency-ext", marker = "extra == 'dev'" }, { name = "nvtx", marker = "extra == 'dev'", specifier = "~=0.2" }, - { name = "nvtx", marker = "extra == 'lts'" }, + { name = "nvtx", marker = "extra == 'lts'", specifier = "~=0.2" }, { name = "onnxscript", marker = "extra == 'dev'" }, + { name = "onnxscript", marker = "extra == 'lts'" }, { name = "opentelemetry-api", marker = "extra == 'dev'", specifier = "~=1.33.1" }, + { name = "opentelemetry-api", marker = "extra == 'lts'", specifier = "~=1.33.1" }, { name = "packaging", specifier = ">=24.2" }, { name = "sentencepiece", marker = "extra == 'mlm'" }, { name = "setuptools", marker = "extra == 'dev'", specifier = "<80.0.0" }, { name = "setuptools", marker = "extra == 'lts'", specifier = "<80.0.0" }, { name = "tensorstore", marker = "extra == 'dev'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, - { name = "tensorstore", marker = "extra == 'lts'", specifier = "!=0.1.46,!=0.1.72" }, + { name = "tensorstore", marker = "extra == 'lts'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, { name = "tiktoken", marker = "extra == 'mlm'" }, { name = "torch" }, { name = "tqdm", marker = "extra == 'dev'" }, { name = "tqdm", marker = "extra == 'lts'" }, { name = "transformer-engine", extras = ["pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.9" }, - { name = "transformers", marker = "extra == 'lts'" }, { name = "transformers", marker = "extra == 'mlm'" }, { name = "wandb", marker = "extra == 'mlm'" }, { name = "wget", marker = "extra == 'dev'" }, { name = "wget", marker = "extra == 'lts'" }, - { name = "zarr", marker = "extra == 'lts'" }, ] provides-extras = ["mlm", "dev", "lts"] @@ -2939,81 +2838,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/90/81ac364ef94209c100e12579629dc92bf7a709a84af32f8c551b02c07e94/nltk-3.9.2-py3-none-any.whl", hash = "sha256:1e209d2b3009110635ed9709a67a1a3e33a10f799490fa71cf4bec218c11c88a", size = 1513404, upload-time = "2025-10-01T07:19:21.648Z" }, ] -[[package]] -name = "numcodecs" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/56/8895a76abe4ec94ebd01eeb6d74f587bc4cddd46569670e1402852a5da13/numcodecs-0.13.1.tar.gz", hash = "sha256:a3cf37881df0898f3a9c0d4477df88133fe85185bffe57ba31bcc2fa207709bc", size = 5955215, upload-time = "2024-10-09T16:28:00.188Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/c0/6d72cde772bcec196b7188731d41282993b2958440f77fdf0db216f722da/numcodecs-0.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:96add4f783c5ce57cc7e650b6cac79dd101daf887c479a00a29bc1487ced180b", size = 1580012, upload-time = "2024-10-09T16:27:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/94/1d/f81fc1fa9210bbea97258242393a1f9feab4f6d8fb201f81f76003005e4b/numcodecs-0.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:237b7171609e868a20fd313748494444458ccd696062f67e198f7f8f52000c15", size = 1176919, upload-time = "2024-10-09T16:27:21.634Z" }, - { url = "https://files.pythonhosted.org/packages/16/e4/b9ec2f4dfc34ecf724bc1beb96a9f6fa9b91801645688ffadacd485089da/numcodecs-0.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96e42f73c31b8c24259c5fac6adba0c3ebf95536e37749dc6c62ade2989dca28", size = 8625842, upload-time = "2024-10-09T16:27:24.168Z" }, - { url = "https://files.pythonhosted.org/packages/fe/90/299952e1477954ec4f92813fa03e743945e3ff711bb4f6c9aace431cb3da/numcodecs-0.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:eda7d7823c9282e65234731fd6bd3986b1f9e035755f7fed248d7d366bb291ab", size = 828638, upload-time = "2024-10-09T16:27:27.063Z" }, - { url = "https://files.pythonhosted.org/packages/f0/78/34b8e869ef143e88d62e8231f4dbfcad85e5c41302a11fc5bd2228a13df5/numcodecs-0.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2eda97dd2f90add98df6d295f2c6ae846043396e3d51a739ca5db6c03b5eb666", size = 1580199, upload-time = "2024-10-09T16:27:29.336Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cf/f70797d86bb585d258d1e6993dced30396f2044725b96ce8bcf87a02be9c/numcodecs-0.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2a86f5367af9168e30f99727ff03b27d849c31ad4522060dde0bce2923b3a8bc", size = 1177203, upload-time = "2024-10-09T16:27:31.011Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b5/d14ad69b63fde041153dfd05d7181a49c0d4864de31a7a1093c8370da957/numcodecs-0.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233bc7f26abce24d57e44ea8ebeb5cd17084690b4e7409dd470fdb75528d615f", size = 8868743, upload-time = "2024-10-09T16:27:32.833Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/27a7b5af0b33f6d61e198faf177fbbf3cb83ff10d9d1a6857b7efc525ad5/numcodecs-0.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:796b3e6740107e4fa624cc636248a1580138b3f1c579160f260f76ff13a4261b", size = 829603, upload-time = "2024-10-09T16:27:35.415Z" }, - { url = "https://files.pythonhosted.org/packages/37/3a/bc09808425e7d3df41e5fc73fc7a802c429ba8c6b05e55f133654ade019d/numcodecs-0.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5195bea384a6428f8afcece793860b1ab0ae28143c853f0b2b20d55a8947c917", size = 1575806, upload-time = "2024-10-09T16:27:37.804Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cc/dc74d0bfdf9ec192332a089d199f1e543e747c556b5659118db7a437dcca/numcodecs-0.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3501a848adaddce98a71a262fee15cd3618312692aa419da77acd18af4a6a3f6", size = 1178233, upload-time = "2024-10-09T16:27:40.169Z" }, - { url = "https://files.pythonhosted.org/packages/d4/ce/434e8e3970b8e92ae9ab6d9db16cb9bc7aa1cd02e17c11de6848224100a1/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53", size = 8857827, upload-time = "2024-10-09T16:27:42.743Z" }, - { url = "https://files.pythonhosted.org/packages/83/e7/1d8b1b266a92f9013c755b1c146c5ad71a2bff147ecbc67f86546a2e4d6a/numcodecs-0.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:e5db4824ebd5389ea30e54bc8aeccb82d514d28b6b68da6c536b8fa4596f4bca", size = 826539, upload-time = "2024-10-09T16:27:44.808Z" }, - { url = "https://files.pythonhosted.org/packages/83/8b/06771dead2cc4a8ae1ea9907737cf1c8d37a323392fa28f938a586373468/numcodecs-0.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a60d75179fd6692e301ddfb3b266d51eb598606dcae7b9fc57f986e8d65cb43", size = 1571660, upload-time = "2024-10-09T16:27:47.125Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ea/d925bf85f92dfe4635356018da9fe4bfecb07b1c72f62b01c1bc47f936b1/numcodecs-0.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f593c7506b0ab248961a3b13cb148cc6e8355662ff124ac591822310bc55ecf", size = 1169925, upload-time = "2024-10-09T16:27:49.512Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d6/643a3839d571d8e439a2c77dc4b0b8cab18d96ac808e4a81dbe88e959ab6/numcodecs-0.13.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d3071465f03522e776a31045ddf2cfee7f52df468b977ed3afdd7fe5869701", size = 8814257, upload-time = "2024-10-09T16:27:52.059Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c5/f3e56bc9b4e438a287fff738993d6d11abef368c0328a612ac2842ba9fca/numcodecs-0.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:90d3065ae74c9342048ae0046006f99dcb1388b7288da5a19b3bddf9c30c3176", size = 821887, upload-time = "2024-10-09T16:27:55.039Z" }, -] - -[[package]] -name = "numcodecs" -version = "0.16.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/48/6188e359b90a9d8a1850f2bc888c023e66f4a8b2b496820babbea414f008/numcodecs-0.16.3.tar.gz", hash = "sha256:53d705865faaf0a7927c973af3777532001c8fbb653de119c1e844608614d799", size = 6275704, upload-time = "2025-09-18T18:54:57.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/cc/917a85972537498f2bbd7914047efc98babc8667587ceb9dcb228378978a/numcodecs-0.16.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:95c9f2a49bef10cf91ad614a761cba9bfe96656b60c12540e1080de5d909b4ca", size = 1642356, upload-time = "2025-09-18T18:54:36.402Z" }, - { url = "https://files.pythonhosted.org/packages/3b/6a/64c25a089e8537441fe67c09ecb7f3f7fb5d98cd04faf01f605d43aca41c/numcodecs-0.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2afe73d5ebaf9ca0cd5c83aad945da80d29a33d860a80d43a7248491d8813ff", size = 1169186, upload-time = "2025-09-18T18:54:37.838Z" }, - { url = "https://files.pythonhosted.org/packages/d8/a0/0de627baeb43e2045a3d4b3de99bf8b69af329a33df1ed4cda468d70c1fb/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:913f08194d82dcb37594e6705e6d4ae6ccd4b6571500b832fb3e4a155de1dfe8", size = 8341668, upload-time = "2025-09-18T18:54:39.444Z" }, - { url = "https://files.pythonhosted.org/packages/b6/0f/49d1f74a216149240c4b9403218111f11670bd11af0919fda357bb056bf2/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a7f1cae9eb18b85709af46570bf9c60056e7155c4c8f610e8080c68124d0e5", size = 8866611, upload-time = "2025-09-18T18:54:41.168Z" }, - { url = "https://files.pythonhosted.org/packages/aa/51/03aece765108fe247717105b5131856546e5428f22a56a14ffdebd017424/numcodecs-0.16.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7bb7f2c46eb7ec8a1c5f8d8fe1a72c222256dd6d6df5af9eaac7a6b905f3575", size = 806787, upload-time = "2025-09-18T18:54:42.78Z" }, - { url = "https://files.pythonhosted.org/packages/0d/78/e4b34803a3aa1d0769919695de4b133266c18c80c474d32ebc462fa1a9bd/numcodecs-0.16.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c77454d92941a335d148b0b822f5d4783103f392774d5d76283bbf7f21b49529", size = 1681108, upload-time = "2025-09-18T18:54:43.856Z" }, - { url = "https://files.pythonhosted.org/packages/25/cf/ca36f463b03a4097767d2a1c1b72f31810e8c6384e9449dd9b925203783c/numcodecs-0.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:270e7a33ee96bdf5c957acf25a2487002a233811a125a155c400c2f036b69c73", size = 1165589, upload-time = "2025-09-18T18:54:44.954Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ae/670260c3c4b5ed34a0674561355f3d4ce7fcbdf09a667e5bc841526d271c/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12f43fa4a347d1dba775c4506a1c9b15b90144c258433b81f79f1c1b1a990db5", size = 8316365, upload-time = "2025-09-18T18:54:46.073Z" }, - { url = "https://files.pythonhosted.org/packages/bb/fa/94e022419c751a60ff0f53642ebae5ef81ed3cc3640f958588e3ad3dc18d/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44869ef564a50aa545215c6a0d42ba5bbc34e9715523fb2336ada3d1fb2b331d", size = 8846228, upload-time = "2025-09-18T18:54:47.858Z" }, - { url = "https://files.pythonhosted.org/packages/71/60/f23733589f3e059bf8589508acd23ffeec230bdf179f138a54f5ab16e0a6/numcodecs-0.16.3-cp312-cp312-win_amd64.whl", hash = "sha256:9aae6996172ba10c5f5111b2998709071b5aeba6b58b1ee0b26b61ed6aa7f2f4", size = 806260, upload-time = "2025-09-18T18:54:49.41Z" }, - { url = "https://files.pythonhosted.org/packages/3c/d5/d3536d06ac1e5fb848a3186958204082b68b106364c9a3669652dd786731/numcodecs-0.16.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:947406b01c20f2ce7ce2e631e7f21b782e8a9d4b57b374a41c9e7b1341a8f3a2", size = 1677129, upload-time = "2025-09-18T18:54:50.5Z" }, - { url = "https://files.pythonhosted.org/packages/e1/fd/b0513a3428dc2b38ec85eea771703ae69c49f09b9650d6c44c9105c80073/numcodecs-0.16.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7cf50e351398a34b45817974c411527629e88937b7683695e276afd65da6ed6f", size = 1159058, upload-time = "2025-09-18T18:54:51.675Z" }, - { url = "https://files.pythonhosted.org/packages/98/05/b7c127283cfb154a97abb284363825401b69302d71a28608af66f73257cc/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7938502fcc060ed9543814f38ca67048b33d7bd2667756e36e6b1060455b17e", size = 8260987, upload-time = "2025-09-18T18:54:52.883Z" }, - { url = "https://files.pythonhosted.org/packages/ff/46/320d960aff884bc63abaaf846ffa3de4803e83e8070b6f84c5688464839c/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:010d628c95be1214536fb22c0df4ced58da954b404b1fcb25ddebf64e4a3f7f3", size = 8805295, upload-time = "2025-09-18T18:54:54.698Z" }, - { url = "https://files.pythonhosted.org/packages/31/ae/acc2e0f1f49ba32afa2174578f170673139248ef86f77e334f2619133867/numcodecs-0.16.3-cp313-cp313-win_amd64.whl", hash = "sha256:e83115e3c32de798c7b7164503e06aae9f9746c1cef564d029616eb44bd6cd90", size = 803204, upload-time = "2025-09-18T18:54:56.192Z" }, -] - -[package.optional-dependencies] -crc32c = [ - { name = "crc32c", marker = "python_full_version >= '3.11'" }, -] - [[package]] name = "numpy" version = "1.26.4" @@ -3057,59 +2881,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/05/79/87c45f32e661b25e0aaa1e325ba166511f57be5dff8f0fcabc12d3e73b64/nv_grouped_gemm-1.1.4.post6.tar.gz", hash = "sha256:dad6115f4b4ff7ceb0bc40ad44e923c13a24fc88cfe1e20b1a6b4c9cf24c445c", size = 26508, upload-time = "2025-10-10T18:52:29.508Z" } -[[package]] -name = "nvidia-cublas-cu12" -version = "12.8.4.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, - { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, - { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208, upload-time = "2025-03-07T01:53:30.535Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, - { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, - { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759, upload-time = "2025-03-07T01:51:11.355Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, - { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, - { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838, upload-time = "2025-03-07T01:52:13.483Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, - { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, - { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318, upload-time = "2025-03-07T01:51:01.794Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu12" -version = "9.10.2.21" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, - { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, - { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268, upload-time = "2025-06-06T21:55:18.114Z" }, -] - [[package]] name = "nvidia-cudnn-frontend" version = "1.15.0" @@ -3129,76 +2900,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/04/519fd6e3ea12fe7fe98c497c4d51f6c5c87763d02e90ea3102cef32a6ef1/nvidia_cudnn_frontend-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:7c8c6f12534b73b0cd55956c5e9419b7840a01e4c260837606112450ce1ca0d9", size = 1297324, upload-time = "2025-10-10T18:46:53.104Z" }, ] -[[package]] -name = "nvidia-cufft-cu12" -version = "11.3.3.83" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, - { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559, upload-time = "2025-03-07T01:53:57.106Z" }, -] - -[[package]] -name = "nvidia-cufile-cu12" -version = "1.13.1.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, -] - -[[package]] -name = "nvidia-curand-cu12" -version = "10.3.9.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, - { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, - { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309, upload-time = "2025-03-07T01:54:20.478Z" }, -] - -[[package]] -name = "nvidia-cusolver-cu12" -version = "11.7.3.90" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, - { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, - { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438, upload-time = "2025-03-07T01:54:39.898Z" }, -] - -[[package]] -name = "nvidia-cusparse-cu12" -version = "12.5.8.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, - { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404, upload-time = "2025-03-07T01:55:07.742Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu12" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, - { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911, upload-time = "2025-02-26T00:14:47.204Z" }, -] - [[package]] name = "nvidia-mathdx" version = "25.1.1" @@ -3240,44 +2941,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/a8/85034a33753a56ef120b931dda1183d98efef1010e379c54d3c3214a5048/nvidia_modelopt-0.37.0-py3-none-any.whl", hash = "sha256:3490b6d6aea3541aa5d475d81230fee627e2c16ff47bbab1cba4b80a1eb119a2", size = 831271, upload-time = "2025-10-08T18:37:03.951Z" }, ] -[[package]] -name = "nvidia-nccl-cu12" -version = "2.27.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/1c/857979db0ef194ca5e21478a0612bcdbbe59458d7694361882279947b349/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a", size = 322400625, upload-time = "2025-06-26T04:11:04.496Z" }, - { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, -] - -[[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710, upload-time = "2025-03-07T01:56:24.13Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu12" -version = "3.3.20" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9d/3dd98852568fb845ec1f7902c90a22b240fe1cbabda411ccedf2fd737b7b/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0", size = 124484616, upload-time = "2025-08-04T20:24:59.172Z" }, - { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, -] - -[[package]] -name = "nvidia-nvtx-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, - { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492, upload-time = "2025-03-07T01:52:24.69Z" }, -] - [[package]] name = "nvidia-resiliency-ext" version = "0.4.1" @@ -3351,8 +3014,8 @@ name = "onnx" version = "1.19.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy" }, { name = "protobuf" }, { name = "typing-extensions" }, @@ -5427,7 +5090,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -5711,30 +5374,15 @@ name = "torch" version = "2.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, - { name = "sympy" }, + { name = "filelock", marker = "sys_platform != 'linux'" }, + { name = "fsspec", marker = "sys_platform != 'linux'" }, + { name = "jinja2", marker = "sys_platform != 'linux'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform != 'linux') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'linux') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "setuptools", marker = "python_full_version >= '3.12' and sys_platform != 'linux'" }, + { name = "sympy", marker = "sys_platform != 'linux'" }, { name = "triton", marker = "sys_platform == 'never'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/bb/86/245c240d2138c17ed572c943c289056c2721abab70810d772c6bf5495b28/torch-2.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:030bbfe367379ae6a4ae4042b6c44da25383343b8b3c68abaa9c7231efbaf2dd", size = 104213554, upload-time = "2025-10-15T15:45:59.798Z" }, @@ -5830,8 +5478,8 @@ dependencies = [ { name = "docstring-parser" }, { name = "filelock" }, { name = "fsspec" }, - { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "pyre-extensions" }, { name = "pyyaml" }, { name = "tabulate" }, @@ -6342,32 +5990,20 @@ name = "wrapt" version = "2.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] sdist = { url = "https://files.pythonhosted.org/packages/49/19/5e5bcd855d808892fe02d49219f97a50f64cd6d8313d75df3494ee97b1a3/wrapt-2.0.0.tar.gz", hash = "sha256:35a542cc7a962331d0279735c30995b024e852cf40481e384fd63caaa391cbb9", size = 81722, upload-time = "2025-10-19T23:47:54.07Z" } wheels = [ @@ -6637,53 +6273,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, ] -[[package]] -name = "zarr" -version = "2.18.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "asciitree", marker = "python_full_version < '3.11'" }, - { name = "fasteners", marker = "python_full_version < '3.11' and sys_platform != 'emscripten'" }, - { name = "numcodecs", version = "0.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/c4/187a21ce7cf7c8f00c060dd0e04c2a81139bb7b1ab178bba83f2e1134ce2/zarr-2.18.3.tar.gz", hash = "sha256:2580d8cb6dd84621771a10d31c4d777dca8a27706a1a89b29f42d2d37e2df5ce", size = 3603224, upload-time = "2024-09-04T23:20:16.595Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/c9/142095e654c2b97133ff71df60979422717b29738b08bc8a1709a5d5e0d0/zarr-2.18.3-py3-none-any.whl", hash = "sha256:b1f7dfd2496f436745cdd4c7bcf8d3b4bc1dceef5fdd0d589c87130d842496dd", size = 210723, upload-time = "2024-09-04T23:20:14.491Z" }, -] - -[[package]] -name = "zarr" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "donfig", marker = "python_full_version >= '3.11'" }, - { name = "numcodecs", version = "0.16.3", source = { registry = "https://pypi.org/simple" }, extra = ["crc32c"], marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/67/14be68a7bad15eecda09b1e81fca2420f7533645fe187bf4d6104c1aad52/zarr-3.1.3.tar.gz", hash = "sha256:01342f3e26a02ed5670db608a5576fbdb8d76acb5c280bd2d0082454b1ba6f79", size = 349125, upload-time = "2025-09-18T19:32:41.688Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/71/9de7229515a53d1cc5705ca9c411530f711a2242f962214d9dbfe2741aa4/zarr-3.1.3-py3-none-any.whl", hash = "sha256:45f67f87f65f14fa453f99dd8110a5936b7ac69f3a21981d33e90407c80c302a", size = 276427, upload-time = "2025-09-18T19:32:40.042Z" }, -] - [[package]] name = "zipp" version = "3.23.0"