diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 95a5dfc86c8..eadc35b9b82 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -232,11 +232,26 @@ jobs: with: python-version: 3.12 + - name: Install GH CLI + shell: bash -x -e -u -o pipefail {0} + run: | + apt-get update + apt-get install -y gh + - name: Get PR info id: get-pr-info if: startsWith(github.ref, 'refs/heads/pull-request/') uses: nv-gha-runners/get-pr-info@main + - name: Has lts label + id: has-lts-label + env: + GH_TOKEN: ${{ secrets.PAT }} + run: | + PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} + HAS_LTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "container::lts")') || echo "false" + echo "main=$HAS_LTS_LABEL" | tee -a $GITHUB_OUTPUT + - name: Download test data shell: bash env: @@ -275,6 +290,22 @@ jobs: echo "$LAST_PRS" | tee -a $GITHUB_OUTPUT echo "EOF" | tee -a $GITHUB_OUTPUT + - name: Parse baseimage + shell: bash + id: base-image + env: + HAS_LTS_LABEL: ${{ steps.has-lts-label.outputs.main }} + run: | + if [ "$HAS_LTS_LABEL" == "true" ]; then + NGC_VERSION=$(cat docker/.ngc_version.lts) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=lts" | tee -a $GITHUB_OUTPUT + else + NGC_VERSION=$(cat docker/.ngc_version.dev) + echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT + echo "image_type=dev" | tee -a $GITHUB_OUTPUT + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -286,7 +317,8 @@ jobs: context: . target: main build-args: | - FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:25.09-py3 + FROM_IMAGE_NAME=${{ steps.base-image.outputs.version }} + IMAGE_TYPE=${{ steps.base-image.outputs.image_type }} cache-from: | type=registry,ref=${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}-buildcache,mode=max type=registry,ref=${{ env.container-registry }}/megatron-lm:main-buildcache,mode=max diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index 0658daaa9ec..d67225311f6 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -53,10 +53,12 @@ test:build_image: parallel: matrix: - IMAGE: CI_MCORE_LTS_IMAGE - FILE: Dockerfile.ci.lts - BASE_IMAGE: nvcr.io/nvidia/pytorch:24.01-py3 + FILE: Dockerfile.ci.dev + IMAGE_TYPE: lts + BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev + IMAGE_TYPE: dev BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.dev @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/.ngc_version.lts b/docker/.ngc_version.lts new file mode 100644 index 00000000000..6b72812b34f --- /dev/null +++ b/docker/.ngc_version.lts @@ -0,0 +1 @@ +nvcr.io/nvidia/pytorch:25.09-py3 \ No newline at end of file diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index 179c2aa21a6..709583d61fc 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -1,8 +1,6 @@ # syntax=docker/dockerfile:1.3-labs ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - FROM ${FROM_IMAGE_NAME} as main ENV PIP_CONSTRAINT="" ENV DEBIAN_FRONTEND=noninteractive @@ -25,16 +23,16 @@ RUN bash -ex <<"EOF" curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh EOF -ARG WHEEL_DIR COPY README.md pyproject.toml uv.lock /workspace/ COPY megatron/core/__init__.py /workspace/megatron/core/ COPY megatron/core/package_info.py /workspace/megatron/core/ +ARG IMAGE_TYPE=dev RUN --mount=type=cache,target=/root/.cache/uv \ bash -ex <<"EOF" export NVTE_CUDA_ARCHS="80;90;100" uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages uv sync --only-group build - uv sync --extra dev --extra mlm --link-mode copy --locked \ + uv sync --extra ${IMAGE_TYPE} --extra mlm --link-mode copy --locked \ --no-install-package torch \ --no-install-package torchvision \ --no-install-package triton \ diff --git a/docker/Dockerfile.ci.lts b/docker/Dockerfile.ci.lts deleted file mode 100644 index 7da27a03f1d..00000000000 --- a/docker/Dockerfile.ci.lts +++ /dev/null @@ -1,98 +0,0 @@ -# syntax=docker/dockerfile:1.3-labs - -ARG FROM_IMAGE_NAME -ARG WHEEL_DIR=/workspace/wheels - -FROM $FROM_IMAGE_NAME as build_mamba -WORKDIR /opt -ARG WHEEL_DIR -RUN MAMBA_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/state-spaces/mamba.git@v2.0.3 -w $WHEEL_DIR - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as build_causalconv1d -WORKDIR /opt -ARG WHEEL_DIR -RUN CAUSAL_CONV1D_FORCE_BUILD=TRUE pip3 wheel -v git+https://github.com/Dao-AILab/causal-conv1d.git@v1.2.2.post1 -w $WHEEL_DIR - -FROM $FROM_IMAGE_NAME as build_groupedgemm -WORKDIR /opt -ARG WHEEL_DIR -RUN pip3 wheel -v git+https://github.com/fanshiqing/grouped_gemm@v1.1.2 -w $WHEEL_DIR - - -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as main -ENV DEBIAN_FRONTEND=noninteractive - -RUN bash -ex <<"EOF" - apt-get update - apt-get install -y --no-install-recommends gettext python3-venv psmisc - apt-get clean - python -m venv /opt/jet - wget https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -O /usr/local/bin/yq - chmod a+x /usr/local/bin/yq -EOF - -ARG UV_VERSION=0.7.2 -ENV PATH="/root/.local/bin:$PATH" -RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh -ENV UV_PROJECT_ENVIRONMENT=/opt/venv -ENV PATH="$UV_PROJECT_ENVIRONMENT/bin:$PATH" -ENV VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT -ENV UV_LINK_MODE=copy - -RUN -ARG WHEEL_DIR -COPY README.md pyproject.toml uv.lock /workspace/ -COPY megatron/core/__init__.py /workspace/megatron/core/ -COPY megatron/core/package_info.py /workspace/megatron/core/ -COPY docker/common/ /workspace/docker/common/ -COPY --from=build_mamba $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_causalconv1d $WHEEL_DIR/*.whl $WHEEL_DIR/ -COPY --from=build_groupedgemm $WHEEL_DIR/*.whl $WHEEL_DIR/ -RUN bash -ex <<"EOF" - uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages - - uv sync --extra lts --extra mlm --link-mode copy --locked \ - --no-install-package torch \ - --no-install-package torchvision \ - --no-install-package triton \ - --no-install-package nvidia-cublas-cu12 \ - --no-install-package nvidia-cuda-cupti-cu12 \ - --no-install-package nvidia-cuda-nvrtc-cu12 \ - --no-install-package nvidia-cuda-runtime-cu12 \ - --no-install-package nvidia-cudnn-cu12 \ - --no-install-package nvidia-cufft-cu12 \ - --no-install-package nvidia-cufile-cu12 \ - --no-install-package nvidia-curand-cu12 \ - --no-install-package nvidia-cusolver-cu12 \ - --no-install-package nvidia-cusparse-cu12 \ - --no-install-package nvidia-cusparselt-cu12 \ - --no-install-package nvidia-nccl-cu12 - - bash docker/common/install_source_wheels.sh --input-wheel-dir $WHEEL_DIR/ --environment lts -EOF -ENV PYTHONPATH="/opt/megatron-lm:$PYTHONPATH" -COPY assets/ /opt/data/ -ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python - -##### For NVIDIANS only ##### -FROM main as jet -ARG JET_API_VERSION -ENV PATH="$PATH:/opt/jet/bin" -RUN --mount=type=secret,id=JET_INDEX_URLS bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - python -m venv /opt/jet - /opt/jet/bin/pip install --no-cache-dir $JET_INDEX_URLS \ - jet-api==$JET_API_VERSION -EOF - -RUN --mount=type=secret,id=JET_INDEX_URLS \ - --mount=type=secret,id=LOGGER_INDEX_URL bash -ex <<"EOF" - JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) - LOGGER_INDEX_URL=$(cat /run/secrets/LOGGER_INDEX_URL) - uv pip install --no-cache-dir --upgrade $LOGGER_INDEX_URL "one-logger" - uv pip install --no-cache-dir --upgrade "setuptools<80.0.0" - uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=3.0" -EOF -### \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3dcfe98a9b7..9dfa88e541e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ mlm = ["flask-restful", "sentencepiece", "tiktoken", "wandb", "transformers"] dev = [ "nvidia-modelopt[torch]; sys_platform != 'darwin'", "transformer-engine[pytorch]>=2.9.0a0,<2.10.0", - "nvidia-resiliency-ext>=0.4.0a0,<0.5.0", + "nvidia-resiliency-ext", "tqdm", "einops~=0.8", "tensorstore~=0.1,!=0.1.46,!=0.1.72", @@ -89,13 +89,20 @@ dev = [ lts = [ "tqdm", - "einops", - "tensorstore!=0.1.46,!=0.1.72", - "nvtx", - "transformers", - "zarr", + "einops~=0.8", + "tensorstore~=0.1,!=0.1.46,!=0.1.72", + "nvtx~=0.2", + "multi-storage-client~=0.27", + "opentelemetry-api~=1.33.1", "setuptools<80.0.0", + "mamba-ssm~=2.2", + "causal-conv1d~=1.5", + "nv-grouped-gemm~=1.1", + "megatron-energon[av_decode]~=6.0", + "av<16.0.0", # At the time, av 16.0.0 is not compatible with Python 3.12 + "flashinfer-python", "wget", + "onnxscript", ] [dependency-groups] diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json new file mode 100644 index 00000000000..6e0bad38133 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84081, + "2": 10.8354, + "3": 10.83237, + "4": 10.81882, + "5": 10.84098, + "6": 10.87094, + "7": 10.83285, + "8": 10.8395, + "9": 10.84275, + "10": 10.80913, + "11": 10.85185, + "12": 10.84426, + "13": 10.86366, + "14": 10.86332, + "15": 10.80028, + "16": 10.79303, + "17": 10.7753, + "18": 10.80133, + "19": 10.79138, + "20": 10.70502, + "21": 10.68161, + "22": 10.56472, + "23": 10.70185, + "24": 10.58, + "25": 10.5355, + "26": 10.607, + "27": 10.59378, + "28": 10.56083, + "29": 10.57494, + "30": 10.35506, + "31": 10.12664, + "32": 10.46551, + "33": 10.45216, + "34": 10.22453, + "35": 10.27096, + "36": 10.22158, + "37": 10.33994, + "38": 10.18651, + "39": 10.39397, + "40": 10.07811, + "41": 10.13811, + "42": 10.20175, + "43": 9.83808, + "44": 9.94297, + "45": 9.82336, + "46": 9.82165, + "47": 10.13421, + "48": 9.84058, + "49": 9.52119, + "50": 9.90123 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1539.0, + "2": 1717.0, + "3": 1773.0, + "4": 1809.0, + "5": 1932.0, + "6": 1836.0, + "7": 1808.0, + "8": 1638.0, + "9": 1924.0, + "10": 1388.0, + "11": 1978.0, + "12": 1875.0, + "13": 1894.0, + "14": 1832.0, + "15": 1945.0, + "16": 1966.0, + "17": 1779.0, + "18": 1731.0, + "19": 1812.0, + "20": 1744.0, + "21": 1910.0, + "22": 1717.0, + "23": 2079.0, + "24": 1636.0, + "25": 1644.0, + "26": 1812.0, + "27": 1939.0, + "28": 1904.0, + "29": 2001.0, + "30": 2019.0, + "31": 1661.0, + "32": 1904.0, + "33": 2040.0, + "34": 1944.0, + "35": 1955.0, + "36": 1968.0, + "37": 2344.0, + "38": 2300.0, + "39": 2418.0, + "40": 2263.0, + "41": 2357.0, + "42": 2285.0, + "43": 1988.0, + "44": 2123.0, + "45": 2218.0, + "46": 2349.0, + "47": 2594.0, + "48": 2506.0, + "49": 2331.0, + "50": 2374.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4576453120.0, + "2": 4673069568.0, + "3": 4673069568.0, + "4": 4673069568.0, + "5": 4673069568.0, + "6": 4673069568.0, + "7": 4673069568.0, + "8": 4673069568.0, + "9": 4673069568.0, + "10": 4673069568.0, + "11": 4673069568.0, + "12": 4673069568.0, + "13": 4673069568.0, + "14": 4673069568.0, + "15": 4673069568.0, + "16": 4673069568.0, + "17": 4673069568.0, + "18": 4673069568.0, + "19": 4673069568.0, + "20": 4673069568.0, + "21": 4673069568.0, + "22": 4673069568.0, + "23": 4673069568.0, + "24": 4673069568.0, + "25": 4673069568.0, + "26": 4673069568.0, + "27": 4673069568.0, + "28": 4673069568.0, + "29": 4673069568.0, + "30": 4673069568.0, + "31": 4673069568.0, + "32": 4673069568.0, + "33": 4673069568.0, + "34": 4673069568.0, + "35": 4673069568.0, + "36": 4673069568.0, + "37": 4673069568.0, + "38": 4673069568.0, + "39": 4673069568.0, + "40": 4673069568.0, + "41": 4673069568.0, + "42": 4673069568.0, + "43": 4673069568.0, + "44": 4673069568.0, + "45": 4673069568.0, + "46": 4673069568.0, + "47": 4673069568.0, + "48": 4673069568.0, + "49": 4673069568.0, + "50": 4673069568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4.12429, + "2": 0.12435, + "3": 0.09661, + "4": 0.09529, + "5": 0.09843, + "6": 0.09665, + "7": 0.09511, + "8": 0.09638, + "9": 0.10162, + "10": 0.10334, + "11": 0.09588, + "12": 0.09381, + "13": 0.09442, + "14": 0.09491, + "15": 0.09381, + "16": 0.09349, + "17": 0.09321, + "18": 0.09347, + "19": 0.09324, + "20": 0.09371, + "21": 0.09293, + "22": 0.09401, + "23": 0.09261, + "24": 0.0939, + "25": 0.09278, + "26": 0.0928, + "27": 0.09252, + "28": 0.09435, + "29": 0.09305, + "30": 0.09435, + "31": 0.0933, + "32": 0.09301, + "33": 0.09463, + "34": 0.09316, + "35": 0.0928, + "36": 0.09341, + "37": 0.09258, + "38": 0.09342, + "39": 0.09405, + "40": 0.0936, + "41": 0.09395, + "42": 0.09327, + "43": 0.09334, + "44": 0.09333, + "45": 0.09265, + "46": 0.09374, + "47": 0.09219, + "48": 0.09286, + "49": 0.09254, + "50": 0.09368 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..cb5f53dccbb --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84454, + "13": 10.8632, + "14": 10.86353, + "15": 10.79985, + "16": 10.79265, + "17": 10.77473, + "18": 10.80156, + "19": 10.79155, + "20": 10.70512, + "21": 10.68174, + "22": 10.56547, + "23": 10.70145, + "24": 10.5789, + "25": 10.53597, + "26": 10.60745, + "27": 10.59418, + "28": 10.56116, + "29": 10.57573, + "30": 10.35474, + "31": 10.12618, + "32": 10.46569, + "33": 10.45235, + "34": 10.22491, + "35": 10.27088, + "36": 10.22167, + "37": 10.33935, + "38": 10.18639, + "39": 10.39432, + "40": 10.07794, + "41": 10.13875, + "42": 10.20184, + "43": 9.83819, + "44": 9.94273, + "45": 9.82299, + "46": 9.82187, + "47": 10.13444, + "48": 9.84097, + "49": 9.52094, + "50": 9.90107, + "51": 9.83459, + "52": 9.73231, + "53": 10.04881, + "54": 9.93895, + "55": 9.86297, + "56": 9.613, + "57": 9.46964, + "58": 9.81136, + "59": 9.57107, + "60": 9.48153, + "61": 9.67881, + "62": 9.96579, + "63": 9.35276, + "64": 9.75644, + "65": 8.93769, + "66": 9.68152, + "67": 9.35669, + "68": 9.76806, + "69": 9.7739, + "70": 9.71012, + "71": 9.60009, + "72": 9.56796, + "73": 9.47739, + "74": 8.93177, + "75": 9.40721, + "76": 9.06847, + "77": 10.0464, + "78": 9.70984, + "79": 9.35731, + "80": 9.38978, + "81": 9.4662, + "82": 9.68056, + "83": 9.29144, + "84": 9.40194, + "85": 9.59734, + "86": 9.06207, + "87": 9.57921, + "88": 9.73262, + "89": 9.58838, + "90": 9.80354, + "91": 9.31991, + "92": 9.35013, + "93": 9.06378, + "94": 8.81909, + "95": 9.50572, + "96": 9.51068, + "97": 9.29244, + "98": 9.65579, + "99": 8.87401, + "100": 9.38837 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1963.0, + "12": 1833.0, + "13": 1863.0, + "14": 1928.0, + "15": 1948.0, + "16": 1999.0, + "17": 1838.0, + "18": 1703.0, + "19": 1898.0, + "20": 1610.0, + "21": 1944.0, + "22": 1728.0, + "23": 2116.0, + "24": 1613.0, + "25": 1634.0, + "26": 1788.0, + "27": 2059.0, + "28": 2067.0, + "29": 1978.0, + "30": 1929.0, + "31": 1782.0, + "32": 1850.0, + "33": 2169.0, + "34": 1815.0, + "35": 2007.0, + "36": 2010.0, + "37": 2385.0, + "38": 2413.0, + "39": 2474.0, + "40": 2254.0, + "41": 2373.0, + "42": 2253.0, + "43": 1900.0, + "44": 2058.0, + "45": 2153.0, + "46": 2385.0, + "47": 2514.0, + "48": 2475.0, + "49": 2362.0, + "50": 2335.0, + "51": 2452.0, + "52": 2576.0, + "53": 2914.0, + "54": 2741.0, + "55": 2408.0, + "56": 2650.0, + "57": 2264.0, + "58": 2853.0, + "59": 2757.0, + "60": 2509.0, + "61": 3076.0, + "62": 2709.0, + "63": 2563.0, + "64": 3041.0, + "65": 2687.0, + "66": 3089.0, + "67": 2767.0, + "68": 2930.0, + "69": 2911.0, + "70": 3286.0, + "71": 3105.0, + "72": 2507.0, + "73": 3063.0, + "74": 2022.0, + "75": 2763.0, + "76": 3002.0, + "77": 3382.0, + "78": 3470.0, + "79": 3109.0, + "80": 3357.0, + "81": 3798.0, + "82": 3348.0, + "83": 2763.0, + "84": 3271.0, + "85": 3245.0, + "86": 2587.0, + "87": 3650.0, + "88": 3103.0, + "89": 3471.0, + "90": 3086.0, + "91": 3050.0, + "92": 3368.0, + "93": 2828.0, + "94": 3495.0, + "95": 3424.0, + "96": 3559.0, + "97": 3289.0, + "98": 3727.0, + "99": 3275.0, + "100": 3401.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 552128512.0, + "2": 552128512.0, + "3": 552128512.0, + "4": 552128512.0, + "5": 552128512.0, + "6": 552128512.0, + "7": 552128512.0, + "8": 552128512.0, + "9": 552128512.0, + "10": 552128512.0, + "11": 552128512.0, + "12": 552128512.0, + "13": 552128512.0, + "14": 552128512.0, + "15": 552128512.0, + "16": 552128512.0, + "17": 552128512.0, + "18": 552128512.0, + "19": 552128512.0, + "20": 552128512.0, + "21": 552128512.0, + "22": 552128512.0, + "23": 552128512.0, + "24": 552128512.0, + "25": 552128512.0, + "26": 552128512.0, + "27": 552128512.0, + "28": 552128512.0, + "29": 552128512.0, + "30": 552128512.0, + "31": 552128512.0, + "32": 552128512.0, + "33": 552128512.0, + "34": 552128512.0, + "35": 552128512.0, + "36": 552128512.0, + "37": 552128512.0, + "38": 552128512.0, + "39": 552128512.0, + "40": 552128512.0, + "41": 552128512.0, + "42": 552128512.0, + "43": 552128512.0, + "44": 552128512.0, + "45": 552128512.0, + "46": 552128512.0, + "47": 552128512.0, + "48": 552128512.0, + "49": 552128512.0, + "50": 552128512.0, + "51": 552128512.0, + "52": 552128512.0, + "53": 552128512.0, + "54": 552128512.0, + "55": 552128512.0, + "56": 552128512.0, + "57": 552128512.0, + "58": 552128512.0, + "59": 552128512.0, + "60": 552128512.0, + "61": 552128512.0, + "62": 552128512.0, + "63": 552128512.0, + "64": 552128512.0, + "65": 552128512.0, + "66": 552128512.0, + "67": 552128512.0, + "68": 552128512.0, + "69": 552128512.0, + "70": 552128512.0, + "71": 552128512.0, + "72": 552128512.0, + "73": 552128512.0, + "74": 552128512.0, + "75": 552128512.0, + "76": 552128512.0, + "77": 552128512.0, + "78": 552128512.0, + "79": 552128512.0, + "80": 552128512.0, + "81": 552128512.0, + "82": 552128512.0, + "83": 552128512.0, + "84": 552128512.0, + "85": 552128512.0, + "86": 552128512.0, + "87": 552128512.0, + "88": 552128512.0, + "89": 552128512.0, + "90": 552128512.0, + "91": 552128512.0, + "92": 552128512.0, + "93": 552128512.0, + "94": 552128512.0, + "95": 552128512.0, + "96": 552128512.0, + "97": 552128512.0, + "98": 552128512.0, + "99": 552128512.0, + "100": 552128512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2615097856.0, + "2": 2711714304.0, + "3": 2711714304.0, + "4": 2711714304.0, + "5": 2711714304.0, + "6": 2711714304.0, + "7": 2711714304.0, + "8": 2711714304.0, + "9": 2711714304.0, + "10": 2711714304.0, + "11": 2711714304.0, + "12": 2711714304.0, + "13": 2711714304.0, + "14": 2711714304.0, + "15": 2711714304.0, + "16": 2711714304.0, + "17": 2711714304.0, + "18": 2711714304.0, + "19": 2711714304.0, + "20": 2711714304.0, + "21": 2711714304.0, + "22": 2711714304.0, + "23": 2711714304.0, + "24": 2711714304.0, + "25": 2711714304.0, + "26": 2711714304.0, + "27": 2711714304.0, + "28": 2711714304.0, + "29": 2711714304.0, + "30": 2711714304.0, + "31": 2711714304.0, + "32": 2711714304.0, + "33": 2711714304.0, + "34": 2711714304.0, + "35": 2711714304.0, + "36": 2711714304.0, + "37": 2711714304.0, + "38": 2711714304.0, + "39": 2711714304.0, + "40": 2711714304.0, + "41": 2711714304.0, + "42": 2711714304.0, + "43": 2711714304.0, + "44": 2711714304.0, + "45": 2711714304.0, + "46": 2711714304.0, + "47": 2711714304.0, + "48": 2711714304.0, + "49": 2711714304.0, + "50": 2711714304.0, + "51": 2711714304.0, + "52": 2711714304.0, + "53": 2711714304.0, + "54": 2711714304.0, + "55": 2711714304.0, + "56": 2711714304.0, + "57": 2711714304.0, + "58": 2711714304.0, + "59": 2711714304.0, + "60": 2711714304.0, + "61": 2711714304.0, + "62": 2711714304.0, + "63": 2711714304.0, + "64": 2711714304.0, + "65": 2711714304.0, + "66": 2711714304.0, + "67": 2711714304.0, + "68": 2711714304.0, + "69": 2711714304.0, + "70": 2711714304.0, + "71": 2711714304.0, + "72": 2711714304.0, + "73": 2711714304.0, + "74": 2711714304.0, + "75": 2711714304.0, + "76": 2711714304.0, + "77": 2711714304.0, + "78": 2711714304.0, + "79": 2711714304.0, + "80": 2711714304.0, + "81": 2711714304.0, + "82": 2711714304.0, + "83": 2711714304.0, + "84": 2711714304.0, + "85": 2711714304.0, + "86": 2711714304.0, + "87": 2711714304.0, + "88": 2711714304.0, + "89": 2711714304.0, + "90": 2711714304.0, + "91": 2711714304.0, + "92": 2711714304.0, + "93": 2711714304.0, + "94": 2711714304.0, + "95": 2711714304.0, + "96": 2711714304.0, + "97": 2711714304.0, + "98": 2711714304.0, + "99": 2711714304.0, + "100": 2711714304.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.06625, + "2": 0.11721, + "3": 0.08793, + "4": 0.08795, + "5": 0.08868, + "6": 0.08815, + "7": 0.08793, + "8": 0.08773, + "9": 0.0865, + "10": 0.08614, + "11": 0.08683, + "12": 0.08676, + "13": 0.08649, + "14": 0.08685, + "15": 0.08581, + "16": 0.08999, + "17": 0.09804, + "18": 0.08574, + "19": 0.08501, + "20": 0.08509, + "21": 0.08416, + "22": 0.08493, + "23": 0.08464, + "24": 0.08427, + "25": 0.08432, + "26": 0.08536, + "27": 0.08512, + "28": 0.08544, + "29": 0.0843, + "30": 0.08556, + "31": 0.08558, + "32": 0.08499, + "33": 0.08416, + "34": 0.08507, + "35": 0.08528, + "36": 0.08499, + "37": 0.08432, + "38": 0.08558, + "39": 0.08449, + "40": 0.08577, + "41": 0.08549, + "42": 0.08515, + "43": 0.08448, + "44": 0.08601, + "45": 0.08468, + "46": 0.08544, + "47": 0.08444, + "48": 0.08558, + "49": 0.09168, + "50": 0.08493, + "51": 0.10229, + "52": 0.09392, + "53": 0.08867, + "54": 0.08586, + "55": 0.08751, + "56": 0.11766, + "57": 0.0858, + "58": 0.08518, + "59": 0.08573, + "60": 0.08514, + "61": 0.08486, + "62": 0.08464, + "63": 0.08577, + "64": 0.08514, + "65": 0.08553, + "66": 0.08489, + "67": 0.08528, + "68": 0.08472, + "69": 0.08519, + "70": 0.08469, + "71": 0.08482, + "72": 0.08511, + "73": 0.08513, + "74": 0.08524, + "75": 0.08472, + "76": 0.08482, + "77": 0.08473, + "78": 0.08579, + "79": 0.08474, + "80": 0.085, + "81": 0.08528, + "82": 0.08505, + "83": 0.08478, + "84": 0.08495, + "85": 0.08481, + "86": 0.08513, + "87": 0.08544, + "88": 0.08555, + "89": 0.08656, + "90": 0.08562, + "91": 0.08512, + "92": 0.08492, + "93": 0.08503, + "94": 0.08502, + "95": 0.08481, + "96": 0.08463, + "97": 0.08542, + "98": 0.08457, + "99": 0.08508, + "100": 0.08507 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json new file mode 100644 index 00000000000..97ced10b43a --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84092, + "2": 10.83661, + "3": 10.83233, + "4": 10.81819, + "5": 10.84059, + "6": 10.86985, + "7": 10.83324, + "8": 10.83877, + "9": 10.84355, + "10": 10.80969, + "11": 10.85186, + "12": 10.84449, + "13": 10.86322, + "14": 10.86353, + "15": 10.79981, + "16": 10.79262, + "17": 10.77477, + "18": 10.80157, + "19": 10.79148, + "20": 10.70508, + "21": 10.68176, + "22": 10.56548, + "23": 10.70147, + "24": 10.57889, + "25": 10.53597, + "26": 10.60742, + "27": 10.59423, + "28": 10.56119, + "29": 10.57569, + "30": 10.35474, + "31": 10.12616, + "32": 10.46566, + "33": 10.45233, + "34": 10.22493, + "35": 10.27091, + "36": 10.22168, + "37": 10.33936, + "38": 10.18641, + "39": 10.39431, + "40": 10.07792, + "41": 10.13872, + "42": 10.20182, + "43": 9.83818, + "44": 9.94274, + "45": 9.82303, + "46": 9.82185, + "47": 10.13443, + "48": 9.84098, + "49": 9.52095, + "50": 9.90109, + "51": 9.83457, + "52": 9.73232, + "53": 10.0488, + "54": 9.93895, + "55": 9.863, + "56": 9.613, + "57": 9.46966, + "58": 9.81135, + "59": 9.57107, + "60": 9.48155, + "61": 9.6788, + "62": 9.96581, + "63": 9.35273, + "64": 9.75648, + "65": 8.93771, + "66": 9.68153, + "67": 9.35671, + "68": 9.76807, + "69": 9.7739, + "70": 9.71016, + "71": 9.60009, + "72": 9.56793, + "73": 9.4774, + "74": 8.93177, + "75": 9.4072, + "76": 9.06849, + "77": 10.0464, + "78": 9.70988, + "79": 9.35733, + "80": 9.38975, + "81": 9.4662, + "82": 9.68058, + "83": 9.2914, + "84": 9.40191, + "85": 9.59735, + "86": 9.06209, + "87": 9.57922, + "88": 9.73259, + "89": 9.58836, + "90": 9.80354, + "91": 9.31991, + "92": 9.35011, + "93": 9.06382, + "94": 8.81909, + "95": 9.50568, + "96": 9.51071, + "97": 9.29241, + "98": 9.65578, + "99": 8.87401, + "100": 9.38833 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1628.0, + "2": 1744.0, + "3": 1662.0, + "4": 1801.0, + "5": 1937.0, + "6": 1866.0, + "7": 1855.0, + "8": 1584.0, + "9": 1901.0, + "10": 1325.0, + "11": 1966.0, + "12": 1767.0, + "13": 1876.0, + "14": 1881.0, + "15": 1928.0, + "16": 1875.0, + "17": 1790.0, + "18": 1736.0, + "19": 1804.0, + "20": 1690.0, + "21": 2008.0, + "22": 1765.0, + "23": 2073.0, + "24": 1618.0, + "25": 1720.0, + "26": 1807.0, + "27": 1861.0, + "28": 2026.0, + "29": 1982.0, + "30": 1981.0, + "31": 1688.0, + "32": 1913.0, + "33": 2123.0, + "34": 1893.0, + "35": 2007.0, + "36": 1987.0, + "37": 2334.0, + "38": 2223.0, + "39": 2417.0, + "40": 2370.0, + "41": 2352.0, + "42": 2269.0, + "43": 1967.0, + "44": 2183.0, + "45": 2150.0, + "46": 2350.0, + "47": 2555.0, + "48": 2463.0, + "49": 2326.0, + "50": 2270.0, + "51": 2508.0, + "52": 2495.0, + "53": 2856.0, + "54": 2692.0, + "55": 2482.0, + "56": 2614.0, + "57": 2283.0, + "58": 2894.0, + "59": 2659.0, + "60": 2561.0, + "61": 3006.0, + "62": 2671.0, + "63": 2488.0, + "64": 3092.0, + "65": 2622.0, + "66": 3108.0, + "67": 2741.0, + "68": 2942.0, + "69": 2983.0, + "70": 3347.0, + "71": 3034.0, + "72": 2438.0, + "73": 3075.0, + "74": 1931.0, + "75": 2722.0, + "76": 2960.0, + "77": 3387.0, + "78": 3268.0, + "79": 3079.0, + "80": 3404.0, + "81": 3674.0, + "82": 3192.0, + "83": 2791.0, + "84": 3224.0, + "85": 3237.0, + "86": 2646.0, + "87": 3840.0, + "88": 3114.0, + "89": 3410.0, + "90": 3184.0, + "91": 3073.0, + "92": 3396.0, + "93": 2711.0, + "94": 3530.0, + "95": 3387.0, + "96": 3530.0, + "97": 3277.0, + "98": 3775.0, + "99": 3421.0, + "100": 3350.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1234585088.0, + "2": 1234585088.0, + "3": 1234585088.0, + "4": 1234585088.0, + "5": 1234585088.0, + "6": 1234585088.0, + "7": 1234585088.0, + "8": 1234585088.0, + "9": 1234585088.0, + "10": 1234585088.0, + "11": 1234585088.0, + "12": 1234585088.0, + "13": 1234585088.0, + "14": 1234585088.0, + "15": 1234585088.0, + "16": 1234585088.0, + "17": 1234585088.0, + "18": 1234585088.0, + "19": 1234585088.0, + "20": 1234585088.0, + "21": 1234585088.0, + "22": 1234585088.0, + "23": 1234585088.0, + "24": 1234585088.0, + "25": 1234585088.0, + "26": 1234585088.0, + "27": 1234585088.0, + "28": 1234585088.0, + "29": 1234585088.0, + "30": 1234585088.0, + "31": 1234585088.0, + "32": 1234585088.0, + "33": 1234585088.0, + "34": 1234585088.0, + "35": 1234585088.0, + "36": 1234585088.0, + "37": 1234585088.0, + "38": 1234585088.0, + "39": 1234585088.0, + "40": 1234585088.0, + "41": 1234585088.0, + "42": 1234585088.0, + "43": 1234585088.0, + "44": 1234585088.0, + "45": 1234585088.0, + "46": 1234585088.0, + "47": 1234585088.0, + "48": 1234585088.0, + "49": 1234585088.0, + "50": 1234585088.0, + "51": 1234585088.0, + "52": 1234585088.0, + "53": 1234585088.0, + "54": 1234585088.0, + "55": 1234585088.0, + "56": 1234585088.0, + "57": 1234585088.0, + "58": 1234585088.0, + "59": 1234585088.0, + "60": 1234585088.0, + "61": 1234585088.0, + "62": 1234585088.0, + "63": 1234585088.0, + "64": 1234585088.0, + "65": 1234585088.0, + "66": 1234585088.0, + "67": 1234585088.0, + "68": 1234585088.0, + "69": 1234585088.0, + "70": 1234585088.0, + "71": 1234585088.0, + "72": 1234585088.0, + "73": 1234585088.0, + "74": 1234585088.0, + "75": 1234585088.0, + "76": 1234585088.0, + "77": 1234585088.0, + "78": 1234585088.0, + "79": 1234585088.0, + "80": 1234585088.0, + "81": 1234585088.0, + "82": 1234585088.0, + "83": 1234585088.0, + "84": 1234585088.0, + "85": 1234585088.0, + "86": 1234585088.0, + "87": 1234585088.0, + "88": 1234585088.0, + "89": 1234585088.0, + "90": 1234585088.0, + "91": 1234585088.0, + "92": 1234585088.0, + "93": 1234585088.0, + "94": 1234585088.0, + "95": 1234585088.0, + "96": 1234585088.0, + "97": 1234585088.0, + "98": 1234585088.0, + "99": 1234585088.0, + "100": 1234585088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1984492544.0, + "2": 2536167424.0, + "3": 2536167424.0, + "4": 2536167424.0, + "5": 2536167424.0, + "6": 2536167424.0, + "7": 2536167424.0, + "8": 2536167424.0, + "9": 2536167424.0, + "10": 2536167424.0, + "11": 2536167424.0, + "12": 2536167424.0, + "13": 2536167424.0, + "14": 2536167424.0, + "15": 2536167424.0, + "16": 2536167424.0, + "17": 2536167424.0, + "18": 2536167424.0, + "19": 2536167424.0, + "20": 2536167424.0, + "21": 2536167424.0, + "22": 2536167424.0, + "23": 2536167424.0, + "24": 2536167424.0, + "25": 2536167424.0, + "26": 2536167424.0, + "27": 2536167424.0, + "28": 2536167424.0, + "29": 2536167424.0, + "30": 2536167424.0, + "31": 2536167424.0, + "32": 2536167424.0, + "33": 2536167424.0, + "34": 2536167424.0, + "35": 2536167424.0, + "36": 2536167424.0, + "37": 2536167424.0, + "38": 2536167424.0, + "39": 2536167424.0, + "40": 2536167424.0, + "41": 2536167424.0, + "42": 2536167424.0, + "43": 2536167424.0, + "44": 2536167424.0, + "45": 2536167424.0, + "46": 2536167424.0, + "47": 2536167424.0, + "48": 2536167424.0, + "49": 2536167424.0, + "50": 2536167424.0, + "51": 2536167424.0, + "52": 2536167424.0, + "53": 2536167424.0, + "54": 2536167424.0, + "55": 2536167424.0, + "56": 2536167424.0, + "57": 2536167424.0, + "58": 2536167424.0, + "59": 2536167424.0, + "60": 2536167424.0, + "61": 2536167424.0, + "62": 2536167424.0, + "63": 2536167424.0, + "64": 2536167424.0, + "65": 2536167424.0, + "66": 2536167424.0, + "67": 2536167424.0, + "68": 2536167424.0, + "69": 2536167424.0, + "70": 2536167424.0, + "71": 2536167424.0, + "72": 2536167424.0, + "73": 2536167424.0, + "74": 2536167424.0, + "75": 2536167424.0, + "76": 2536167424.0, + "77": 2536167424.0, + "78": 2536167424.0, + "79": 2536167424.0, + "80": 2536167424.0, + "81": 2536167424.0, + "82": 2536167424.0, + "83": 2536167424.0, + "84": 2536167424.0, + "85": 2536167424.0, + "86": 2536167424.0, + "87": 2536167424.0, + "88": 2536167424.0, + "89": 2536167424.0, + "90": 2536167424.0, + "91": 2536167424.0, + "92": 2536167424.0, + "93": 2536167424.0, + "94": 2536167424.0, + "95": 2536167424.0, + "96": 2536167424.0, + "97": 2536167424.0, + "98": 2536167424.0, + "99": 2536167424.0, + "100": 2536167424.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.49803, + "2": 0.14641, + "3": 0.1267, + "4": 0.12591, + "5": 0.12655, + "6": 0.12634, + "7": 0.12653, + "8": 0.12441, + "9": 0.12554, + "10": 0.12517, + "11": 0.12429, + "12": 0.12458, + "13": 0.12434, + "14": 0.125, + "15": 0.12506, + "16": 0.12228, + "17": 0.12197, + "18": 0.12216, + "19": 0.12297, + "20": 0.12196, + "21": 0.13521, + "22": 0.12613, + "23": 0.12326, + "24": 0.12164, + "25": 0.12372, + "26": 0.12163, + "27": 0.12379, + "28": 0.1211, + "29": 0.12303, + "30": 0.12188, + "31": 0.12303, + "32": 0.12151, + "33": 0.12255, + "34": 0.12109, + "35": 0.12342, + "36": 0.12127, + "37": 0.12271, + "38": 0.12107, + "39": 0.12322, + "40": 0.12093, + "41": 0.12622, + "42": 0.12505, + "43": 0.12728, + "44": 0.12408, + "45": 0.12542, + "46": 0.12407, + "47": 0.12514, + "48": 0.12394, + "49": 0.12534, + "50": 0.12306, + "51": 0.12831, + "52": 0.12326, + "53": 0.12341, + "54": 0.12189, + "55": 0.12225, + "56": 0.12185, + "57": 0.12275, + "58": 0.12252, + "59": 0.12219, + "60": 0.1214, + "61": 0.12242, + "62": 0.12203, + "63": 0.12239, + "64": 0.12186, + "65": 0.12197, + "66": 0.12148, + "67": 0.12172, + "68": 0.12215, + "69": 0.12209, + "70": 0.12183, + "71": 0.12162, + "72": 0.12205, + "73": 0.12233, + "74": 0.12204, + "75": 0.12251, + "76": 0.12231, + "77": 0.12147, + "78": 0.12227, + "79": 0.12192, + "80": 0.12237, + "81": 0.122, + "82": 0.12206, + "83": 0.12237, + "84": 0.12152, + "85": 0.1219, + "86": 0.12229, + "87": 0.12198, + "88": 0.12183, + "89": 0.12274, + "90": 0.12137, + "91": 0.12204, + "92": 0.12204, + "93": 0.12169, + "94": 0.12379, + "95": 0.12103, + "96": 0.12109, + "97": 0.12039, + "98": 0.12225, + "99": 0.12393, + "100": 0.12171 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json new file mode 100644 index 00000000000..8665206969f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84865, + "2": 10.84835, + "3": 10.84054, + "4": 10.82381, + "5": 10.86283, + "6": 10.87554, + "7": 10.87089, + "8": 10.85554, + "9": 10.86811, + "10": 10.82287, + "11": 10.90334, + "12": 10.87985, + "13": 10.8831, + "14": 10.89653, + "15": 10.81406, + "16": 10.81973, + "17": 10.79918, + "18": 10.82428, + "19": 10.8195, + "20": 10.71958, + "21": 10.67864, + "22": 10.53625, + "23": 10.72125, + "24": 10.57435, + "25": 10.52997, + "26": 10.60096, + "27": 10.61502, + "28": 10.57231, + "29": 10.58614, + "30": 10.33749, + "31": 10.06516, + "32": 10.46436, + "33": 10.43612, + "34": 10.17278, + "35": 10.24173, + "36": 10.19042, + "37": 10.32282, + "38": 10.14881, + "39": 10.37709, + "40": 10.05124, + "41": 10.11355, + "42": 10.17253, + "43": 9.76298, + "44": 9.89293, + "45": 9.7664, + "46": 9.7601, + "47": 10.09424, + "48": 9.78753, + "49": 9.454, + "50": 9.8548, + "51": 9.79157, + "52": 9.68731, + "53": 10.02181, + "54": 9.90398, + "55": 9.82389, + "56": 9.57081, + "57": 9.40818, + "58": 9.77678, + "59": 9.52729, + "60": 9.44284, + "61": 9.64071, + "62": 9.94046, + "63": 9.31099, + "64": 9.72506, + "65": 8.8916, + "66": 9.6525, + "67": 9.31718, + "68": 9.73957, + "69": 9.74304, + "70": 9.67942, + "71": 9.56228, + "72": 9.53149, + "73": 9.44531, + "74": 8.88431, + "75": 9.3677, + "76": 9.02482, + "77": 10.01647, + "78": 9.6813, + "79": 9.32719, + "80": 9.3577, + "81": 9.43335, + "82": 9.64804, + "83": 9.25573, + "84": 9.36738, + "85": 9.56091, + "86": 9.03567, + "87": 9.54622, + "88": 9.70041, + "89": 9.54992, + "90": 9.77126, + "91": 9.28801, + "92": 9.31055, + "93": 9.03195, + "94": 8.78121, + "95": 9.48115, + "96": 9.4759, + "97": 9.2489, + "98": 9.61705, + "99": 8.8368, + "100": 9.35043 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1658.0, + "2": 1892.0, + "3": 1844.0, + "4": 1851.0, + "5": 1895.0, + "6": 2058.0, + "7": 1902.0, + "8": 1808.0, + "9": 1894.0, + "10": 1583.0, + "11": 2044.0, + "12": 1889.0, + "13": 2101.0, + "14": 2062.0, + "15": 1981.0, + "16": 1922.0, + "17": 1845.0, + "18": 1842.0, + "19": 1890.0, + "20": 1769.0, + "21": 2048.0, + "22": 1800.0, + "23": 2134.0, + "24": 1822.0, + "25": 1861.0, + "26": 1864.0, + "27": 1886.0, + "28": 2050.0, + "29": 2009.0, + "30": 2118.0, + "31": 1688.0, + "32": 2112.0, + "33": 2153.0, + "34": 1963.0, + "35": 2014.0, + "36": 2023.0, + "37": 2222.0, + "38": 2239.0, + "39": 2367.0, + "40": 2161.0, + "41": 2422.0, + "42": 2240.0, + "43": 2116.0, + "44": 2343.0, + "45": 2127.0, + "46": 2189.0, + "47": 2411.0, + "48": 2347.0, + "49": 2271.0, + "50": 2345.0, + "51": 2482.0, + "52": 2570.0, + "53": 2835.0, + "54": 2589.0, + "55": 2450.0, + "56": 2744.0, + "57": 2429.0, + "58": 2684.0, + "59": 2748.0, + "60": 2464.0, + "61": 2995.0, + "62": 2518.0, + "63": 2570.0, + "64": 2843.0, + "65": 2648.0, + "66": 2842.0, + "67": 2954.0, + "68": 2833.0, + "69": 3027.0, + "70": 2993.0, + "71": 3010.0, + "72": 2597.0, + "73": 3002.0, + "74": 2325.0, + "75": 2882.0, + "76": 3143.0, + "77": 3062.0, + "78": 3272.0, + "79": 3303.0, + "80": 3280.0, + "81": 3517.0, + "82": 3283.0, + "83": 2834.0, + "84": 3365.0, + "85": 3288.0, + "86": 2562.0, + "87": 3493.0, + "88": 3388.0, + "89": 3102.0, + "90": 3230.0, + "91": 3154.0, + "92": 3263.0, + "93": 2967.0, + "94": 3520.0, + "95": 3175.0, + "96": 3317.0, + "97": 2999.0, + "98": 3549.0, + "99": 3248.0, + "100": 3227.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 892293120.0, + "2": 892293120.0, + "3": 892293120.0, + "4": 892293120.0, + "5": 892293120.0, + "6": 892293120.0, + "7": 892293120.0, + "8": 892293120.0, + "9": 892293120.0, + "10": 892293120.0, + "11": 892293120.0, + "12": 892293120.0, + "13": 892293120.0, + "14": 892293120.0, + "15": 892293120.0, + "16": 892293120.0, + "17": 892293120.0, + "18": 892293120.0, + "19": 892293120.0, + "20": 892293120.0, + "21": 892293120.0, + "22": 892293120.0, + "23": 892293120.0, + "24": 892293120.0, + "25": 892293120.0, + "26": 892293120.0, + "27": 892293120.0, + "28": 892293120.0, + "29": 892293120.0, + "30": 892293120.0, + "31": 892293120.0, + "32": 892293120.0, + "33": 892293120.0, + "34": 892293120.0, + "35": 892293120.0, + "36": 892293120.0, + "37": 892293120.0, + "38": 892293120.0, + "39": 892293120.0, + "40": 892293120.0, + "41": 892293120.0, + "42": 892293120.0, + "43": 892293120.0, + "44": 892293120.0, + "45": 892293120.0, + "46": 892293120.0, + "47": 892293120.0, + "48": 892293120.0, + "49": 892293120.0, + "50": 892293120.0, + "51": 892293120.0, + "52": 892293120.0, + "53": 892293120.0, + "54": 892293120.0, + "55": 892293120.0, + "56": 892293120.0, + "57": 892293120.0, + "58": 892293120.0, + "59": 892293120.0, + "60": 892293120.0, + "61": 892293120.0, + "62": 892293120.0, + "63": 892293120.0, + "64": 892293120.0, + "65": 892293120.0, + "66": 892293120.0, + "67": 892293120.0, + "68": 892293120.0, + "69": 892293120.0, + "70": 892293120.0, + "71": 892293120.0, + "72": 892293120.0, + "73": 892293120.0, + "74": 892293120.0, + "75": 892293120.0, + "76": 892293120.0, + "77": 892293120.0, + "78": 892293120.0, + "79": 892293120.0, + "80": 892293120.0, + "81": 892293120.0, + "82": 892293120.0, + "83": 892293120.0, + "84": 892293120.0, + "85": 892293120.0, + "86": 892293120.0, + "87": 892293120.0, + "88": 892293120.0, + "89": 892293120.0, + "90": 892293120.0, + "91": 892293120.0, + "92": 892293120.0, + "93": 892293120.0, + "94": 892293120.0, + "95": 892293120.0, + "96": 892293120.0, + "97": 892293120.0, + "98": 892293120.0, + "99": 892293120.0, + "100": 892293120.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2236674048.0, + "2": 2599285760.0, + "3": 2599285760.0, + "4": 2599285760.0, + "5": 2599285760.0, + "6": 2599285760.0, + "7": 2599285760.0, + "8": 2599285760.0, + "9": 2599285760.0, + "10": 2599285760.0, + "11": 2599285760.0, + "12": 2599285760.0, + "13": 2599285760.0, + "14": 2599285760.0, + "15": 2599285760.0, + "16": 2599285760.0, + "17": 2599285760.0, + "18": 2599285760.0, + "19": 2599285760.0, + "20": 2599285760.0, + "21": 2599285760.0, + "22": 2599285760.0, + "23": 2599285760.0, + "24": 2599285760.0, + "25": 2599285760.0, + "26": 2599285760.0, + "27": 2599285760.0, + "28": 2599285760.0, + "29": 2599285760.0, + "30": 2599285760.0, + "31": 2599285760.0, + "32": 2599285760.0, + "33": 2599285760.0, + "34": 2599285760.0, + "35": 2599285760.0, + "36": 2599285760.0, + "37": 2599285760.0, + "38": 2599285760.0, + "39": 2599285760.0, + "40": 2599285760.0, + "41": 2599285760.0, + "42": 2599285760.0, + "43": 2599285760.0, + "44": 2599285760.0, + "45": 2599285760.0, + "46": 2599285760.0, + "47": 2599285760.0, + "48": 2599285760.0, + "49": 2599285760.0, + "50": 2599285760.0, + "51": 2599285760.0, + "52": 2599285760.0, + "53": 2599285760.0, + "54": 2599285760.0, + "55": 2599285760.0, + "56": 2599285760.0, + "57": 2599285760.0, + "58": 2599285760.0, + "59": 2599285760.0, + "60": 2599285760.0, + "61": 2599285760.0, + "62": 2599285760.0, + "63": 2599285760.0, + "64": 2599285760.0, + "65": 2599285760.0, + "66": 2599285760.0, + "67": 2599285760.0, + "68": 2599285760.0, + "69": 2599285760.0, + "70": 2599285760.0, + "71": 2599285760.0, + "72": 2599285760.0, + "73": 2599285760.0, + "74": 2599285760.0, + "75": 2599285760.0, + "76": 2599285760.0, + "77": 2599285760.0, + "78": 2599285760.0, + "79": 2599285760.0, + "80": 2599285760.0, + "81": 2599285760.0, + "82": 2599285760.0, + "83": 2599285760.0, + "84": 2599285760.0, + "85": 2599285760.0, + "86": 2599285760.0, + "87": 2599285760.0, + "88": 2599285760.0, + "89": 2599285760.0, + "90": 2599285760.0, + "91": 2599285760.0, + "92": 2599285760.0, + "93": 2599285760.0, + "94": 2599285760.0, + "95": 2599285760.0, + "96": 2599285760.0, + "97": 2599285760.0, + "98": 2599285760.0, + "99": 2599285760.0, + "100": 2599285760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.4674, + "2": 0.20384, + "3": 0.15442, + "4": 0.14865, + "5": 0.14873, + "6": 0.149, + "7": 0.14827, + "8": 0.14846, + "9": 0.14976, + "10": 0.1489, + "11": 0.14911, + "12": 0.14903, + "13": 0.14943, + "14": 0.14869, + "15": 0.14998, + "16": 0.1487, + "17": 0.14875, + "18": 0.14864, + "19": 0.14923, + "20": 0.15005, + "21": 0.14888, + "22": 0.14908, + "23": 0.14918, + "24": 0.14933, + "25": 0.1494, + "26": 0.1492, + "27": 0.14942, + "28": 0.1509, + "29": 0.14857, + "30": 0.1482, + "31": 0.14918, + "32": 0.15584, + "33": 0.14904, + "34": 0.14831, + "35": 0.14865, + "36": 0.14946, + "37": 0.151, + "38": 0.14997, + "39": 0.14932, + "40": 0.1486, + "41": 0.14892, + "42": 0.14879, + "43": 0.14926, + "44": 0.14872, + "45": 0.14986, + "46": 0.14856, + "47": 0.14914, + "48": 0.14908, + "49": 0.14863, + "50": 0.14941, + "51": 0.15887, + "52": 0.15717, + "53": 0.15562, + "54": 0.15607, + "55": 0.15613, + "56": 0.15542, + "57": 0.15563, + "58": 0.15517, + "59": 0.15529, + "60": 0.15578, + "61": 0.15428, + "62": 0.15497, + "63": 0.15495, + "64": 0.15513, + "65": 0.15521, + "66": 0.15538, + "67": 0.15487, + "68": 0.15587, + "69": 0.15582, + "70": 0.15554, + "71": 0.15476, + "72": 0.15462, + "73": 0.15206, + "74": 0.15177, + "75": 0.15248, + "76": 0.15299, + "77": 0.15181, + "78": 0.15184, + "79": 0.15197, + "80": 0.15194, + "81": 0.15291, + "82": 0.15283, + "83": 0.1518, + "84": 0.15229, + "85": 0.15204, + "86": 0.153, + "87": 0.15209, + "88": 0.15242, + "89": 0.15239, + "90": 0.15203, + "91": 0.15287, + "92": 0.1524, + "93": 0.15219, + "94": 0.15298, + "95": 0.15257, + "96": 0.15245, + "97": 0.15297, + "98": 0.15209, + "99": 0.15334, + "100": 0.15305 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json new file mode 100644 index 00000000000..5e03e1d9965 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79332, + "2": 10.80674, + "3": 10.8069, + "4": 10.77377, + "5": 10.84763, + "6": 10.86617, + "7": 10.8277, + "8": 10.81226, + "9": 10.83507, + "10": 10.77007, + "11": 10.89243, + "12": 10.84575, + "13": 10.85895, + "14": 10.883, + "15": 10.79054, + "16": 10.78048, + "17": 10.75666, + "18": 10.79491, + "19": 10.79656, + "20": 10.68058, + "21": 10.66001, + "22": 10.50257, + "23": 10.7118, + "24": 10.55327, + "25": 10.50697, + "26": 10.58286, + "27": 10.58643, + "28": 10.55758, + "29": 10.56227, + "30": 10.33257, + "31": 10.08654, + "32": 10.44724, + "33": 10.44499, + "34": 10.19999, + "35": 10.25465, + "36": 10.19443, + "37": 10.32044, + "38": 10.16641, + "39": 10.3774, + "40": 10.05603, + "41": 10.13739, + "42": 10.19161, + "43": 9.80954, + "44": 9.93054, + "45": 9.80619, + "46": 9.81395, + "47": 10.12881, + "48": 9.82729, + "49": 9.51291, + "50": 9.89126, + "51": 9.84055, + "52": 9.73438, + "53": 10.05482, + "54": 9.94058, + "55": 9.87124, + "56": 9.61045, + "57": 9.46116, + "58": 9.81654, + "59": 9.57887, + "60": 9.48507, + "61": 9.68515, + "62": 9.97438, + "63": 9.36298, + "64": 9.76793, + "65": 8.93913, + "66": 9.68918, + "67": 9.36638, + "68": 9.77507, + "69": 9.78344, + "70": 9.72196, + "71": 9.60806, + "72": 9.57714, + "73": 9.48934, + "74": 8.94008, + "75": 9.40867, + "76": 9.08075, + "77": 10.05717, + "78": 9.72281, + "79": 9.36465, + "80": 9.39746, + "81": 9.47553, + "82": 9.6886, + "83": 9.30263, + "84": 9.41008, + "85": 9.60793, + "86": 9.07115, + "87": 9.58676, + "88": 9.74129, + "89": 9.5986, + "90": 9.81041, + "91": 9.33113, + "92": 9.35502, + "93": 9.07481, + "94": 8.82745, + "95": 9.51149, + "96": 9.51876, + "97": 9.30173, + "98": 9.66726, + "99": 8.88087, + "100": 9.39727 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1624.0, + "2": 1758.0, + "3": 1678.0, + "4": 1680.0, + "5": 1824.0, + "6": 1899.0, + "7": 1871.0, + "8": 1677.0, + "9": 1802.0, + "10": 1414.0, + "11": 1904.0, + "12": 1630.0, + "13": 1956.0, + "14": 1777.0, + "15": 1902.0, + "16": 1801.0, + "17": 1830.0, + "18": 1630.0, + "19": 1835.0, + "20": 1642.0, + "21": 1752.0, + "22": 1696.0, + "23": 2085.0, + "24": 1618.0, + "25": 1593.0, + "26": 1722.0, + "27": 1778.0, + "28": 2042.0, + "29": 1900.0, + "30": 2001.0, + "31": 1592.0, + "32": 1757.0, + "33": 2116.0, + "34": 1924.0, + "35": 1904.0, + "36": 1852.0, + "37": 2382.0, + "38": 2195.0, + "39": 2267.0, + "40": 2335.0, + "41": 2223.0, + "42": 2317.0, + "43": 2069.0, + "44": 2060.0, + "45": 2140.0, + "46": 2397.0, + "47": 2464.0, + "48": 2455.0, + "49": 2276.0, + "50": 2374.0, + "51": 2574.0, + "52": 2457.0, + "53": 2905.0, + "54": 2609.0, + "55": 2220.0, + "56": 2663.0, + "57": 2258.0, + "58": 2898.0, + "59": 2676.0, + "60": 2397.0, + "61": 3048.0, + "62": 2533.0, + "63": 2370.0, + "64": 2975.0, + "65": 2591.0, + "66": 3065.0, + "67": 2732.0, + "68": 2870.0, + "69": 2955.0, + "70": 3112.0, + "71": 2989.0, + "72": 2451.0, + "73": 2881.0, + "74": 1859.0, + "75": 2649.0, + "76": 3026.0, + "77": 3316.0, + "78": 3212.0, + "79": 3183.0, + "80": 3262.0, + "81": 3669.0, + "82": 3187.0, + "83": 2798.0, + "84": 3209.0, + "85": 3309.0, + "86": 2738.0, + "87": 3804.0, + "88": 2989.0, + "89": 3327.0, + "90": 3031.0, + "91": 2720.0, + "92": 2972.0, + "93": 2719.0, + "94": 3387.0, + "95": 3321.0, + "96": 3342.0, + "97": 3191.0, + "98": 3533.0, + "99": 3214.0, + "100": 3318.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 716834304.0, + "2": 716834304.0, + "3": 716834304.0, + "4": 716834304.0, + "5": 716834304.0, + "6": 716834304.0, + "7": 716834304.0, + "8": 716834304.0, + "9": 716834304.0, + "10": 716834304.0, + "11": 716834304.0, + "12": 716834304.0, + "13": 716834304.0, + "14": 716834304.0, + "15": 716834304.0, + "16": 716834304.0, + "17": 716834304.0, + "18": 716834304.0, + "19": 716834304.0, + "20": 716834304.0, + "21": 716834304.0, + "22": 716834304.0, + "23": 716834304.0, + "24": 716834304.0, + "25": 716834304.0, + "26": 716834304.0, + "27": 716834304.0, + "28": 716834304.0, + "29": 716834304.0, + "30": 716834304.0, + "31": 716834304.0, + "32": 716834304.0, + "33": 716834304.0, + "34": 716834304.0, + "35": 716834304.0, + "36": 716834304.0, + "37": 716834304.0, + "38": 716834304.0, + "39": 716834304.0, + "40": 716834304.0, + "41": 716834304.0, + "42": 716834304.0, + "43": 716834304.0, + "44": 716834304.0, + "45": 716834304.0, + "46": 716834304.0, + "47": 716834304.0, + "48": 716834304.0, + "49": 716834304.0, + "50": 716834304.0, + "51": 716834304.0, + "52": 716834304.0, + "53": 716834304.0, + "54": 716834304.0, + "55": 716834304.0, + "56": 716834304.0, + "57": 716834304.0, + "58": 716834304.0, + "59": 716834304.0, + "60": 716834304.0, + "61": 716834304.0, + "62": 716834304.0, + "63": 716834304.0, + "64": 716834304.0, + "65": 716834304.0, + "66": 716834304.0, + "67": 716834304.0, + "68": 716834304.0, + "69": 716834304.0, + "70": 716834304.0, + "71": 716834304.0, + "72": 716834304.0, + "73": 716834304.0, + "74": 716834304.0, + "75": 716834304.0, + "76": 716834304.0, + "77": 716834304.0, + "78": 716834304.0, + "79": 716834304.0, + "80": 716834304.0, + "81": 716834304.0, + "82": 716834304.0, + "83": 716834304.0, + "84": 716834304.0, + "85": 716834304.0, + "86": 716834304.0, + "87": 716834304.0, + "88": 716834304.0, + "89": 716834304.0, + "90": 716834304.0, + "91": 716834304.0, + "92": 716834304.0, + "93": 716834304.0, + "94": 716834304.0, + "95": 716834304.0, + "96": 716834304.0, + "97": 716834304.0, + "98": 716834304.0, + "99": 716834304.0, + "100": 716834304.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910424576.0, + "2": 2193074176.0, + "3": 2193074176.0, + "4": 2193074176.0, + "5": 2193074176.0, + "6": 2193074176.0, + "7": 2193074176.0, + "8": 2193074176.0, + "9": 2193074176.0, + "10": 2193074176.0, + "11": 2193074176.0, + "12": 2193074176.0, + "13": 2193074176.0, + "14": 2193074176.0, + "15": 2193074176.0, + "16": 2193074176.0, + "17": 2193074176.0, + "18": 2193074176.0, + "19": 2193074176.0, + "20": 2193074176.0, + "21": 2193074176.0, + "22": 2193074176.0, + "23": 2193074176.0, + "24": 2193074176.0, + "25": 2193074176.0, + "26": 2193074176.0, + "27": 2193074176.0, + "28": 2193074176.0, + "29": 2193074176.0, + "30": 2193074176.0, + "31": 2193074176.0, + "32": 2193074176.0, + "33": 2193074176.0, + "34": 2193074176.0, + "35": 2193074176.0, + "36": 2193074176.0, + "37": 2193074176.0, + "38": 2193074176.0, + "39": 2193074176.0, + "40": 2193074176.0, + "41": 2193074176.0, + "42": 2193074176.0, + "43": 2193074176.0, + "44": 2193074176.0, + "45": 2193074176.0, + "46": 2193074176.0, + "47": 2193074176.0, + "48": 2193074176.0, + "49": 2193074176.0, + "50": 2193074176.0, + "51": 2193074176.0, + "52": 2193074176.0, + "53": 2193074176.0, + "54": 2193074176.0, + "55": 2193074176.0, + "56": 2193074176.0, + "57": 2193074176.0, + "58": 2193074176.0, + "59": 2193074176.0, + "60": 2193074176.0, + "61": 2193074176.0, + "62": 2193074176.0, + "63": 2193074176.0, + "64": 2193074176.0, + "65": 2193074176.0, + "66": 2193074176.0, + "67": 2193074176.0, + "68": 2193074176.0, + "69": 2193074176.0, + "70": 2193074176.0, + "71": 2193074176.0, + "72": 2193074176.0, + "73": 2193074176.0, + "74": 2193074176.0, + "75": 2193074176.0, + "76": 2193074176.0, + "77": 2193074176.0, + "78": 2193074176.0, + "79": 2193074176.0, + "80": 2193074176.0, + "81": 2193074176.0, + "82": 2193074176.0, + "83": 2193074176.0, + "84": 2193074176.0, + "85": 2193074176.0, + "86": 2193074176.0, + "87": 2193074176.0, + "88": 2193074176.0, + "89": 2193074176.0, + "90": 2193074176.0, + "91": 2193074176.0, + "92": 2193074176.0, + "93": 2193074176.0, + "94": 2193074176.0, + "95": 2193074176.0, + "96": 2193074176.0, + "97": 2193074176.0, + "98": 2193074176.0, + "99": 2193074176.0, + "100": 2193074176.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.1813, + "2": 0.1835, + "3": 0.15572, + "4": 0.15355, + "5": 0.151, + "6": 0.14921, + "7": 0.14734, + "8": 0.14714, + "9": 0.14809, + "10": 0.1484, + "11": 0.14826, + "12": 0.14787, + "13": 0.14757, + "14": 0.14755, + "15": 0.14777, + "16": 0.14865, + "17": 0.14838, + "18": 0.14829, + "19": 0.14882, + "20": 0.14865, + "21": 0.14787, + "22": 0.1484, + "23": 0.14894, + "24": 0.14776, + "25": 0.14794, + "26": 0.14803, + "27": 0.14825, + "28": 0.14808, + "29": 0.14832, + "30": 0.14775, + "31": 0.14899, + "32": 0.14815, + "33": 0.14712, + "34": 0.14539, + "35": 0.17395, + "36": 0.14838, + "37": 0.14687, + "38": 0.17567, + "39": 0.14724, + "40": 0.14754, + "41": 0.14861, + "42": 0.14832, + "43": 0.14888, + "44": 0.14802, + "45": 0.14786, + "46": 0.14917, + "47": 0.14831, + "48": 0.14818, + "49": 0.14762, + "50": 0.14759, + "51": 0.15481, + "52": 0.1495, + "53": 0.17829, + "54": 0.14739, + "55": 0.14906, + "56": 0.14669, + "57": 0.14714, + "58": 0.14784, + "59": 0.14791, + "60": 0.14683, + "61": 0.1469, + "62": 0.14708, + "63": 0.14774, + "64": 0.14678, + "65": 0.14768, + "66": 0.14729, + "67": 0.14797, + "68": 0.14707, + "69": 0.14726, + "70": 0.14748, + "71": 0.14624, + "72": 0.1472, + "73": 0.14758, + "74": 0.14695, + "75": 0.1472, + "76": 0.14699, + "77": 0.14601, + "78": 0.1475, + "79": 0.14664, + "80": 0.14662, + "81": 0.14732, + "82": 0.14753, + "83": 0.14694, + "84": 0.14745, + "85": 0.14692, + "86": 0.14819, + "87": 0.14694, + "88": 0.14673, + "89": 0.14753, + "90": 0.14687, + "91": 0.14673, + "92": 0.14638, + "93": 0.14835, + "94": 0.14753, + "95": 0.14756, + "96": 0.14655, + "97": 0.14754, + "98": 0.14729, + "99": 0.1466, + "100": 0.14681 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json new file mode 100644 index 00000000000..1f5abc70e7c --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.90314, + "2": 10.88897, + "3": 10.89725, + "4": 10.88248, + "5": 10.89447, + "6": 10.91226, + "7": 10.89775, + "8": 10.88643, + "9": 10.89227, + "10": 10.8849, + "11": 10.91413, + "12": 10.88666, + "13": 10.89283, + "14": 10.90486, + "15": 10.83878, + "16": 10.84927, + "17": 10.83248, + "18": 10.83371, + "19": 10.83359, + "20": 10.74074, + "21": 10.7043, + "22": 10.59893, + "23": 10.72222, + "24": 10.60745, + "25": 10.57467, + "26": 10.62999, + "27": 10.62364, + "28": 10.57229, + "29": 10.6073, + "30": 10.37766, + "31": 10.15362, + "32": 10.47609, + "33": 10.48062, + "34": 10.24216, + "35": 10.29035, + "36": 10.25955, + "37": 10.36145, + "38": 10.21396, + "39": 10.44502, + "40": 10.11492, + "41": 10.1605, + "42": 10.23468, + "43": 9.85032, + "44": 9.97764, + "45": 9.85681, + "46": 9.8307, + "47": 10.17976, + "48": 9.85811, + "49": 9.54378, + "50": 9.93469, + "51": 9.86793, + "52": 9.76274, + "53": 10.10895, + "54": 9.95538, + "55": 9.8756, + "56": 9.64751, + "57": 9.48989, + "58": 9.85502, + "59": 9.59457, + "60": 9.52968, + "61": 9.69589, + "62": 10.01676, + "63": 9.38778, + "64": 9.80211, + "65": 8.95119, + "66": 9.72857, + "67": 9.37577, + "68": 9.80463, + "69": 9.81, + "70": 9.7662, + "71": 9.63135, + "72": 9.5784, + "73": 9.52148, + "74": 8.94976, + "75": 9.43087, + "76": 9.08489, + "77": 10.089, + "78": 9.72754, + "79": 9.37612, + "80": 9.40849, + "81": 9.49766, + "82": 9.71298, + "83": 9.33332, + "84": 9.43928, + "85": 9.63373, + "86": 9.07038, + "87": 9.61245, + "88": 9.78304, + "89": 9.60878, + "90": 9.85164, + "91": 9.34542, + "92": 9.38281, + "93": 9.07319, + "94": 8.81684, + "95": 9.51809, + "96": 9.54033, + "97": 9.34061, + "98": 9.70134, + "99": 8.88786, + "100": 9.43285 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727000.0, + "2": 22925616.0, + "3": 22596924.0, + "4": 23219540.0, + "5": 22714814.0, + "6": 23021786.0, + "7": 22771458.0, + "8": 22926012.0, + "9": 22842856.0, + "10": 22918360.0, + "11": 22500702.0, + "12": 22459866.0, + "13": 22916820.0, + "14": 22389026.0, + "15": 22821260.0, + "16": 22830812.0, + "17": 22818944.0, + "18": 22582240.0, + "19": 22618380.0, + "20": 22694352.0, + "21": 22740180.0, + "22": 22800024.0, + "23": 22540132.0, + "24": 22771492.0, + "25": 22818970.0, + "26": 22546852.0, + "27": 22468504.0, + "28": 22453052.0, + "29": 22529222.0, + "30": 22631432.0, + "31": 22955696.0, + "32": 22585238.0, + "33": 22557676.0, + "34": 22835412.0, + "35": 22788032.0, + "36": 22589678.0, + "37": 22497140.0, + "38": 22896132.0, + "39": 22801314.0, + "40": 22658064.0, + "41": 22659700.0, + "42": 22667816.0, + "43": 22976356.0, + "44": 22746708.0, + "45": 22675272.0, + "46": 22884382.0, + "47": 22634556.0, + "48": 22928080.0, + "49": 22727538.0, + "50": 22905284.0, + "51": 22791326.0, + "52": 22749392.0, + "53": 22925970.0, + "54": 22839434.0, + "55": 22518416.0, + "56": 22877660.0, + "57": 23113304.0, + "58": 22845008.0, + "59": 22715512.0, + "60": 22743058.0, + "61": 22723950.0, + "62": 22673248.0, + "63": 22846074.0, + "64": 22823228.0, + "65": 23060212.0, + "66": 22729902.0, + "67": 22907278.0, + "68": 22610092.0, + "69": 22584360.0, + "70": 22829348.0, + "71": 22749420.0, + "72": 22655446.0, + "73": 22740974.0, + "74": 23048296.0, + "75": 23053922.0, + "76": 22901008.0, + "77": 22272806.0, + "78": 22789370.0, + "79": 22743288.0, + "80": 22706236.0, + "81": 22890976.0, + "82": 22777092.0, + "83": 22839240.0, + "84": 23010352.0, + "85": 22712004.0, + "86": 23103740.0, + "87": 22734788.0, + "88": 22637620.0, + "89": 22499200.0, + "90": 22972420.0, + "91": 22766428.0, + "92": 22808890.0, + "93": 22659888.0, + "94": 22910970.0, + "95": 23048514.0, + "96": 22829470.0, + "97": 22608826.0, + "98": 22763528.0, + "99": 22905754.0, + "100": 23016268.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 719180288.0, + "2": 719180288.0, + "3": 719180288.0, + "4": 719180288.0, + "5": 719180288.0, + "6": 719180288.0, + "7": 719180288.0, + "8": 719180288.0, + "9": 719180288.0, + "10": 719180288.0, + "11": 719180288.0, + "12": 719180288.0, + "13": 719180288.0, + "14": 719180288.0, + "15": 719180288.0, + "16": 719180288.0, + "17": 719180288.0, + "18": 719180288.0, + "19": 719180288.0, + "20": 719180288.0, + "21": 719180288.0, + "22": 719180288.0, + "23": 719180288.0, + "24": 719180288.0, + "25": 719180288.0, + "26": 719180288.0, + "27": 719180288.0, + "28": 719180288.0, + "29": 719180288.0, + "30": 719180288.0, + "31": 719180288.0, + "32": 719180288.0, + "33": 719180288.0, + "34": 719180288.0, + "35": 719180288.0, + "36": 719180288.0, + "37": 719180288.0, + "38": 719180288.0, + "39": 719180288.0, + "40": 719180288.0, + "41": 719180288.0, + "42": 719180288.0, + "43": 719180288.0, + "44": 719180288.0, + "45": 719180288.0, + "46": 719180288.0, + "47": 719180288.0, + "48": 719180288.0, + "49": 719180288.0, + "50": 719180288.0, + "51": 719180288.0, + "52": 719180288.0, + "53": 719180288.0, + "54": 719180288.0, + "55": 719180288.0, + "56": 719180288.0, + "57": 719180288.0, + "58": 719180288.0, + "59": 719180288.0, + "60": 719180288.0, + "61": 719180288.0, + "62": 719180288.0, + "63": 719180288.0, + "64": 719180288.0, + "65": 719180288.0, + "66": 719180288.0, + "67": 719180288.0, + "68": 719180288.0, + "69": 719180288.0, + "70": 719180288.0, + "71": 719180288.0, + "72": 719180288.0, + "73": 719180288.0, + "74": 719180288.0, + "75": 719180288.0, + "76": 719180288.0, + "77": 719180288.0, + "78": 719180288.0, + "79": 719180288.0, + "80": 719180288.0, + "81": 719180288.0, + "82": 719180288.0, + "83": 719180288.0, + "84": 719180288.0, + "85": 719180288.0, + "86": 719180288.0, + "87": 719180288.0, + "88": 719180288.0, + "89": 719180288.0, + "90": 719180288.0, + "91": 719180288.0, + "92": 719180288.0, + "93": 719180288.0, + "94": 719180288.0, + "95": 719180288.0, + "96": 719180288.0, + "97": 719180288.0, + "98": 719180288.0, + "99": 719180288.0, + "100": 719180288.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1910562816.0, + "2": 2195420160.0, + "3": 2195420160.0, + "4": 2195420160.0, + "5": 2195420160.0, + "6": 2195420160.0, + "7": 2195420160.0, + "8": 2195420160.0, + "9": 2195420160.0, + "10": 2195420160.0, + "11": 2195420160.0, + "12": 2195420160.0, + "13": 2195420160.0, + "14": 2195420160.0, + "15": 2195420160.0, + "16": 2195420160.0, + "17": 2195420160.0, + "18": 2195420160.0, + "19": 2195420160.0, + "20": 2195420160.0, + "21": 2195420160.0, + "22": 2195420160.0, + "23": 2195420160.0, + "24": 2195420160.0, + "25": 2195420160.0, + "26": 2195420160.0, + "27": 2195420160.0, + "28": 2195420160.0, + "29": 2195420160.0, + "30": 2195420160.0, + "31": 2195420160.0, + "32": 2195420160.0, + "33": 2195420160.0, + "34": 2195420160.0, + "35": 2195420160.0, + "36": 2195420160.0, + "37": 2195420160.0, + "38": 2195420160.0, + "39": 2195420160.0, + "40": 2195420160.0, + "41": 2195420160.0, + "42": 2195420160.0, + "43": 2195420160.0, + "44": 2195420160.0, + "45": 2195420160.0, + "46": 2195420160.0, + "47": 2195420160.0, + "48": 2195420160.0, + "49": 2195420160.0, + "50": 2195420160.0, + "51": 2195420160.0, + "52": 2195420160.0, + "53": 2195420160.0, + "54": 2195420160.0, + "55": 2195420160.0, + "56": 2195420160.0, + "57": 2195420160.0, + "58": 2195420160.0, + "59": 2195420160.0, + "60": 2195420160.0, + "61": 2195420160.0, + "62": 2195420160.0, + "63": 2195420160.0, + "64": 2195420160.0, + "65": 2195420160.0, + "66": 2195420160.0, + "67": 2195420160.0, + "68": 2195420160.0, + "69": 2195420160.0, + "70": 2195420160.0, + "71": 2195420160.0, + "72": 2195420160.0, + "73": 2195420160.0, + "74": 2195420160.0, + "75": 2195420160.0, + "76": 2195420160.0, + "77": 2195420160.0, + "78": 2195420160.0, + "79": 2195420160.0, + "80": 2195420160.0, + "81": 2195420160.0, + "82": 2195420160.0, + "83": 2195420160.0, + "84": 2195420160.0, + "85": 2195420160.0, + "86": 2195420160.0, + "87": 2195420160.0, + "88": 2195420160.0, + "89": 2195420160.0, + "90": 2195420160.0, + "91": 2195420160.0, + "92": 2195420160.0, + "93": 2195420160.0, + "94": 2195420160.0, + "95": 2195420160.0, + "96": 2195420160.0, + "97": 2195420160.0, + "98": 2195420160.0, + "99": 2195420160.0, + "100": 2195420160.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.95459, + "2": 0.18402, + "3": 0.15999, + "4": 0.15839, + "5": 0.15834, + "6": 0.15838, + "7": 0.15707, + "8": 0.15667, + "9": 0.15683, + "10": 0.15803, + "11": 0.15755, + "12": 0.15791, + "13": 0.15599, + "14": 0.15788, + "15": 0.15809, + "16": 0.1578, + "17": 0.15771, + "18": 0.1566, + "19": 0.1561, + "20": 0.15522, + "21": 0.15579, + "22": 0.15567, + "23": 0.15642, + "24": 0.15624, + "25": 0.15571, + "26": 0.15742, + "27": 0.15594, + "28": 0.15677, + "29": 0.15668, + "30": 0.15543, + "31": 0.15641, + "32": 0.1563, + "33": 0.15652, + "34": 0.15591, + "35": 0.15642, + "36": 0.1562, + "37": 0.15572, + "38": 0.1573, + "39": 0.15622, + "40": 0.15669, + "41": 0.15823, + "42": 0.15583, + "43": 0.15743, + "44": 0.15635, + "45": 0.15626, + "46": 0.15674, + "47": 0.1569, + "48": 0.15693, + "49": 0.15676, + "50": 0.15573, + "51": 0.16368, + "52": 0.1561, + "53": 0.15657, + "54": 0.15564, + "55": 0.15503, + "56": 0.15438, + "57": 0.15511, + "58": 0.1538, + "59": 0.15446, + "60": 0.15384, + "61": 0.15369, + "62": 0.15299, + "63": 0.15496, + "64": 0.15487, + "65": 0.15436, + "66": 0.15351, + "67": 0.15426, + "68": 0.15447, + "69": 0.15414, + "70": 0.15343, + "71": 0.15372, + "72": 0.15462, + "73": 0.15443, + "74": 0.15437, + "75": 0.15414, + "76": 0.15474, + "77": 0.15388, + "78": 0.15486, + "79": 0.15403, + "80": 0.15416, + "81": 0.15422, + "82": 0.15445, + "83": 0.15379, + "84": 0.15405, + "85": 0.15441, + "86": 0.15347, + "87": 0.15361, + "88": 0.15401, + "89": 0.15431, + "90": 0.15341, + "91": 0.15446, + "92": 0.15397, + "93": 0.15443, + "94": 0.15423, + "95": 0.15396, + "96": 0.15478, + "97": 0.15338, + "98": 0.15446, + "99": 0.15405, + "100": 0.15554 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json new file mode 100644 index 00000000000..88ed0c3c716 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79513, + "16": 10.7958, + "17": 10.76832, + "18": 10.8109, + "19": 10.79887, + "20": 10.69131, + "21": 10.6801, + "22": 10.52152, + "23": 10.7071, + "24": 10.57678, + "25": 10.52316, + "26": 10.59563, + "27": 10.58607, + "28": 10.56175, + "29": 10.56945, + "30": 10.34623, + "31": 10.10035, + "32": 10.45432, + "33": 10.44591, + "34": 10.2072, + "35": 10.26185, + "36": 10.21228, + "37": 10.32449, + "38": 10.16803, + "39": 10.38353, + "40": 10.07219, + "41": 10.13754, + "42": 10.19755, + "43": 9.81134, + "44": 9.93287, + "45": 9.80998, + "46": 9.80859, + "47": 10.12583, + "48": 9.82132, + "49": 9.50738, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1894.0, + "16": 1731.0, + "17": 1848.0, + "18": 1705.0, + "19": 1686.0, + "20": 1620.0, + "21": 1840.0, + "22": 1764.0, + "23": 1937.0, + "24": 1620.0, + "25": 1728.0, + "26": 1727.0, + "27": 1821.0, + "28": 2042.0, + "29": 2029.0, + "30": 1825.0, + "31": 1594.0, + "32": 1903.0, + "33": 2041.0, + "34": 1895.0, + "35": 1908.0, + "36": 1906.0, + "37": 2224.0, + "38": 2150.0, + "39": 2327.0, + "40": 2074.0, + "41": 2314.0, + "42": 2230.0, + "43": 1920.0, + "44": 2115.0, + "45": 1962.0, + "46": 2287.0, + "47": 2481.0, + "48": 2407.0, + "49": 2270.0, + "50": 2349.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3837453824.0, + "2": 4119165440.0, + "3": 4119165440.0, + "4": 4119165440.0, + "5": 4119165440.0, + "6": 4119165440.0, + "7": 4119165440.0, + "8": 4119165440.0, + "9": 4119165440.0, + "10": 4119165440.0, + "11": 4119165440.0, + "12": 4119165440.0, + "13": 4119165440.0, + "14": 4119165440.0, + "15": 4119165440.0, + "16": 4119165440.0, + "17": 4119165440.0, + "18": 4119165440.0, + "19": 4119165440.0, + "20": 4119165440.0, + "21": 4119165440.0, + "22": 4119165440.0, + "23": 4119165440.0, + "24": 4119165440.0, + "25": 4119165440.0, + "26": 4119165440.0, + "27": 4119165440.0, + "28": 4119165440.0, + "29": 4119165440.0, + "30": 4119165440.0, + "31": 4119165440.0, + "32": 4119165440.0, + "33": 4119165440.0, + "34": 4119165440.0, + "35": 4119165440.0, + "36": 4119165440.0, + "37": 4119165440.0, + "38": 4119165440.0, + "39": 4119165440.0, + "40": 4119165440.0, + "41": 4119165440.0, + "42": 4119165440.0, + "43": 4119165440.0, + "44": 4119165440.0, + "45": 4119165440.0, + "46": 4119165440.0, + "47": 4119165440.0, + "48": 4119165440.0, + "49": 4119165440.0, + "50": 4119165440.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.33649, + "2": 0.17516, + "3": 0.14384, + "4": 0.13957, + "5": 0.13869, + "6": 0.13892, + "7": 0.1372, + "8": 0.13599, + "9": 0.13689, + "10": 0.13673, + "11": 0.13594, + "12": 0.1351, + "13": 0.13665, + "14": 0.13582, + "15": 0.13559, + "16": 0.136, + "17": 0.13575, + "18": 0.13545, + "19": 0.13608, + "20": 0.13614, + "21": 0.13687, + "22": 0.14045, + "23": 0.13788, + "24": 0.13963, + "25": 0.13971, + "26": 0.13813, + "27": 0.13825, + "28": 0.13906, + "29": 0.14053, + "30": 0.13933, + "31": 0.13846, + "32": 0.13642, + "33": 0.1344, + "34": 0.13566, + "35": 0.13538, + "36": 0.13529, + "37": 0.13419, + "38": 0.13511, + "39": 0.13541, + "40": 0.13526, + "41": 0.13623, + "42": 0.13487, + "43": 0.13619, + "44": 0.13477, + "45": 0.13423, + "46": 0.13533, + "47": 0.13712, + "48": 0.13255, + "49": 0.13243, + "50": 0.1331 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json new file mode 100644 index 00000000000..e15024aee55 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81478, + "2": 10.82042, + "3": 10.81232, + "4": 10.78901, + "5": 10.85168, + "6": 10.87098, + "7": 10.83086, + "8": 10.83472, + "9": 10.83886, + "10": 10.78749, + "11": 10.87935, + "12": 10.86017, + "13": 10.86578, + "14": 10.8781, + "15": 10.79514, + "16": 10.79576, + "17": 10.76832, + "18": 10.81092, + "19": 10.79889, + "20": 10.69131, + "21": 10.68008, + "22": 10.52146, + "23": 10.70713, + "24": 10.57677, + "25": 10.52315, + "26": 10.59564, + "27": 10.5861, + "28": 10.56176, + "29": 10.56942, + "30": 10.34624, + "31": 10.10032, + "32": 10.45433, + "33": 10.44592, + "34": 10.20725, + "35": 10.26186, + "36": 10.21223, + "37": 10.32453, + "38": 10.16801, + "39": 10.38354, + "40": 10.07222, + "41": 10.13752, + "42": 10.19756, + "43": 9.81134, + "44": 9.93285, + "45": 9.81001, + "46": 9.80858, + "47": 10.12582, + "48": 9.82129, + "49": 9.50739, + "50": 9.88351 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1544.0, + "2": 1712.0, + "3": 1660.0, + "4": 1767.0, + "5": 1795.0, + "6": 1799.0, + "7": 1805.0, + "8": 1664.0, + "9": 1769.0, + "10": 1340.0, + "11": 1830.0, + "12": 1812.0, + "13": 1837.0, + "14": 1710.0, + "15": 1839.0, + "16": 1776.0, + "17": 1750.0, + "18": 1612.0, + "19": 1764.0, + "20": 1649.0, + "21": 1854.0, + "22": 1750.0, + "23": 1909.0, + "24": 1616.0, + "25": 1654.0, + "26": 1755.0, + "27": 1860.0, + "28": 2042.0, + "29": 1953.0, + "30": 1905.0, + "31": 1684.0, + "32": 1831.0, + "33": 2101.0, + "34": 1769.0, + "35": 1915.0, + "36": 1885.0, + "37": 2324.0, + "38": 2169.0, + "39": 2300.0, + "40": 2069.0, + "41": 2353.0, + "42": 2236.0, + "43": 1978.0, + "44": 2022.0, + "45": 2103.0, + "46": 2292.0, + "47": 2413.0, + "48": 2305.0, + "49": 2218.0, + "50": 2321.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 523364864.0, + "2": 523364864.0, + "3": 523364864.0, + "4": 523364864.0, + "5": 523364864.0, + "6": 523364864.0, + "7": 523364864.0, + "8": 523364864.0, + "9": 523364864.0, + "10": 523364864.0, + "11": 523364864.0, + "12": 523364864.0, + "13": 523364864.0, + "14": 523364864.0, + "15": 523364864.0, + "16": 523364864.0, + "17": 523364864.0, + "18": 523364864.0, + "19": 523364864.0, + "20": 523364864.0, + "21": 523364864.0, + "22": 523364864.0, + "23": 523364864.0, + "24": 523364864.0, + "25": 523364864.0, + "26": 523364864.0, + "27": 523364864.0, + "28": 523364864.0, + "29": 523364864.0, + "30": 523364864.0, + "31": 523364864.0, + "32": 523364864.0, + "33": 523364864.0, + "34": 523364864.0, + "35": 523364864.0, + "36": 523364864.0, + "37": 523364864.0, + "38": 523364864.0, + "39": 523364864.0, + "40": 523364864.0, + "41": 523364864.0, + "42": 523364864.0, + "43": 523364864.0, + "44": 523364864.0, + "45": 523364864.0, + "46": 523364864.0, + "47": 523364864.0, + "48": 523364864.0, + "49": 523364864.0, + "50": 523364864.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3768846848.0, + "2": 3914174976.0, + "3": 3914174976.0, + "4": 3914174976.0, + "5": 3914174976.0, + "6": 3914174976.0, + "7": 3914174976.0, + "8": 3914174976.0, + "9": 3914174976.0, + "10": 3914174976.0, + "11": 3914174976.0, + "12": 3914174976.0, + "13": 3914174976.0, + "14": 3914174976.0, + "15": 3914174976.0, + "16": 3914174976.0, + "17": 3914174976.0, + "18": 3914174976.0, + "19": 3914174976.0, + "20": 3914174976.0, + "21": 3914174976.0, + "22": 3914174976.0, + "23": 3914174976.0, + "24": 3914174976.0, + "25": 3914174976.0, + "26": 3914174976.0, + "27": 3914174976.0, + "28": 3914174976.0, + "29": 3914174976.0, + "30": 3914174976.0, + "31": 3914174976.0, + "32": 3914174976.0, + "33": 3914174976.0, + "34": 3914174976.0, + "35": 3914174976.0, + "36": 3914174976.0, + "37": 3914174976.0, + "38": 3914174976.0, + "39": 3914174976.0, + "40": 3914174976.0, + "41": 3914174976.0, + "42": 3914174976.0, + "43": 3914174976.0, + "44": 3914174976.0, + "45": 3914174976.0, + "46": 3914174976.0, + "47": 3914174976.0, + "48": 3914174976.0, + "49": 3914174976.0, + "50": 3914174976.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.4283, + "2": 0.18086, + "3": 0.15536, + "4": 0.1518, + "5": 0.14978, + "6": 0.15021, + "7": 0.14912, + "8": 0.14691, + "9": 0.14665, + "10": 0.14636, + "11": 0.2893, + "12": 0.14563, + "13": 0.14695, + "14": 0.14631, + "15": 0.1462, + "16": 0.14611, + "17": 0.14601, + "18": 0.14618, + "19": 0.14548, + "20": 0.14665, + "21": 0.29161, + "22": 0.14654, + "23": 0.1467, + "24": 0.14617, + "25": 0.14634, + "26": 0.14707, + "27": 0.14673, + "28": 0.14597, + "29": 0.14632, + "30": 0.14748, + "31": 0.29068, + "32": 0.14752, + "33": 0.14788, + "34": 0.14732, + "35": 0.14747, + "36": 0.14736, + "37": 0.14706, + "38": 0.14766, + "39": 0.14678, + "40": 0.14656, + "41": 0.29211, + "42": 0.14505, + "43": 0.14731, + "44": 0.14467, + "45": 0.14411, + "46": 0.14463, + "47": 0.14361, + "48": 0.14383, + "49": 0.1443, + "50": 0.14467 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json new file mode 100644 index 00000000000..f5f43f44c0f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81271, + "4": 10.78877, + "5": 10.85169, + "6": 10.8704, + "7": 10.831, + "8": 10.83427, + "9": 10.83995, + "10": 10.78684, + "11": 10.88021, + "12": 10.85971, + "13": 10.86589, + "14": 10.87818, + "15": 10.79463, + "16": 10.79607, + "17": 10.7688, + "18": 10.81045, + "19": 10.79836, + "20": 10.69045, + "21": 10.67932, + "22": 10.52101, + "23": 10.70743, + "24": 10.57665, + "25": 10.52275, + "26": 10.595, + "27": 10.5855, + "28": 10.56131, + "29": 10.56894, + "30": 10.34527, + "31": 10.10019, + "32": 10.45229, + "33": 10.44356, + "34": 10.20397, + "35": 10.25844, + "36": 10.2103, + "37": 10.32252, + "38": 10.1661, + "39": 10.38156, + "40": 10.07025, + "41": 10.13542, + "42": 10.19416, + "43": 9.80626, + "44": 9.92627, + "45": 9.8024, + "46": 9.79983, + "47": 10.11662, + "48": 9.81307, + "49": 9.50044, + "50": 9.87631, + "51": 9.82781, + "52": 9.71723, + "53": 10.03979, + "54": 9.92177, + "55": 9.85515, + "56": 9.59253, + "57": 9.44144, + "58": 9.79602, + "59": 9.55567, + "60": 9.4664, + "61": 9.6666, + "62": 9.95363, + "63": 9.33626, + "64": 9.74152, + "65": 8.9178, + "66": 9.66632, + "67": 9.34424, + "68": 9.75273, + "69": 9.75727, + "70": 9.69242, + "71": 9.5868, + "72": 9.55099, + "73": 9.46289, + "74": 8.90671, + "75": 9.37793, + "76": 9.04952, + "77": 10.0301, + "78": 9.69192, + "79": 9.33464, + "80": 9.3667, + "81": 9.44418, + "82": 9.66164, + "83": 9.27209, + "84": 9.38066, + "85": 9.57618, + "86": 9.0424, + "87": 9.55703, + "88": 9.70385, + "89": 9.56619, + "90": 9.77295, + "91": 9.29396, + "92": 9.31912, + "93": 9.03406, + "94": 8.78526, + "95": 9.46938, + "96": 9.47497, + "97": 9.25688, + "98": 9.61835, + "99": 8.83233, + "100": 9.34557 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1668.0, + "4": 1721.0, + "5": 1897.0, + "6": 1823.0, + "7": 1719.0, + "8": 1637.0, + "9": 1742.0, + "10": 1358.0, + "11": 1882.0, + "12": 1781.0, + "13": 1847.0, + "14": 1753.0, + "15": 1883.0, + "16": 1755.0, + "17": 1752.0, + "18": 1683.0, + "19": 1817.0, + "20": 1663.0, + "21": 1795.0, + "22": 1698.0, + "23": 1996.0, + "24": 1620.0, + "25": 1658.0, + "26": 1727.0, + "27": 1781.0, + "28": 2085.0, + "29": 1952.0, + "30": 1821.0, + "31": 1646.0, + "32": 1879.0, + "33": 2034.0, + "34": 1861.0, + "35": 1834.0, + "36": 1913.0, + "37": 2333.0, + "38": 2070.0, + "39": 2245.0, + "40": 2126.0, + "41": 2311.0, + "42": 2213.0, + "43": 1907.0, + "44": 1951.0, + "45": 2001.0, + "46": 2218.0, + "47": 2533.0, + "48": 2436.0, + "49": 2188.0, + "50": 2342.0, + "51": 2562.0, + "52": 2529.0, + "53": 3031.0, + "54": 2744.0, + "55": 2264.0, + "56": 2794.0, + "57": 2183.0, + "58": 2882.0, + "59": 2769.0, + "60": 2399.0, + "61": 3031.0, + "62": 2706.0, + "63": 2388.0, + "64": 3046.0, + "65": 2597.0, + "66": 3092.0, + "67": 2730.0, + "68": 2858.0, + "69": 2982.0, + "70": 3202.0, + "71": 2964.0, + "72": 2450.0, + "73": 2817.0, + "74": 1834.0, + "75": 2609.0, + "76": 3000.0, + "77": 3180.0, + "78": 3113.0, + "79": 3145.0, + "80": 3258.0, + "81": 3645.0, + "82": 3075.0, + "83": 2812.0, + "84": 3295.0, + "85": 3368.0, + "86": 2730.0, + "87": 3717.0, + "88": 3056.0, + "89": 3252.0, + "90": 2954.0, + "91": 2798.0, + "92": 3089.0, + "93": 2742.0, + "94": 3420.0, + "95": 3225.0, + "96": 3362.0, + "97": 3118.0, + "98": 3671.0, + "99": 3341.0, + "100": 3428.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2366437376.0, + "2": 2648148992.0, + "3": 2648148992.0, + "4": 2648148992.0, + "5": 2648148992.0, + "6": 2648148992.0, + "7": 2648148992.0, + "8": 2648148992.0, + "9": 2648148992.0, + "10": 2648148992.0, + "11": 2648148992.0, + "12": 2648148992.0, + "13": 2648148992.0, + "14": 2648148992.0, + "15": 2648148992.0, + "16": 2648148992.0, + "17": 2648148992.0, + "18": 2648148992.0, + "19": 2648148992.0, + "20": 2648148992.0, + "21": 2648148992.0, + "22": 2648148992.0, + "23": 2648148992.0, + "24": 2648148992.0, + "25": 2648148992.0, + "26": 2648148992.0, + "27": 2648148992.0, + "28": 2648148992.0, + "29": 2648148992.0, + "30": 2648148992.0, + "31": 2648148992.0, + "32": 2648148992.0, + "33": 2648148992.0, + "34": 2648148992.0, + "35": 2648148992.0, + "36": 2648148992.0, + "37": 2648148992.0, + "38": 2648148992.0, + "39": 2648148992.0, + "40": 2648148992.0, + "41": 2648148992.0, + "42": 2648148992.0, + "43": 2648148992.0, + "44": 2648148992.0, + "45": 2648148992.0, + "46": 2648148992.0, + "47": 2648148992.0, + "48": 2648148992.0, + "49": 2648148992.0, + "50": 2648148992.0, + "51": 2648148992.0, + "52": 2648148992.0, + "53": 2648148992.0, + "54": 2648148992.0, + "55": 2648148992.0, + "56": 2648148992.0, + "57": 2648148992.0, + "58": 2648148992.0, + "59": 2648148992.0, + "60": 2648148992.0, + "61": 2648148992.0, + "62": 2648148992.0, + "63": 2648148992.0, + "64": 2648148992.0, + "65": 2648148992.0, + "66": 2648148992.0, + "67": 2648148992.0, + "68": 2648148992.0, + "69": 2648148992.0, + "70": 2648148992.0, + "71": 2648148992.0, + "72": 2648148992.0, + "73": 2648148992.0, + "74": 2648148992.0, + "75": 2648148992.0, + "76": 2648148992.0, + "77": 2648148992.0, + "78": 2648148992.0, + "79": 2648148992.0, + "80": 2648148992.0, + "81": 2648148992.0, + "82": 2648148992.0, + "83": 2648148992.0, + "84": 2648148992.0, + "85": 2648148992.0, + "86": 2648148992.0, + "87": 2648148992.0, + "88": 2648148992.0, + "89": 2648148992.0, + "90": 2648148992.0, + "91": 2648148992.0, + "92": 2648148992.0, + "93": 2648148992.0, + "94": 2648148992.0, + "95": 2648148992.0, + "96": 2648148992.0, + "97": 2648148992.0, + "98": 2648148992.0, + "99": 2648148992.0, + "100": 2648148992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.83862, + "2": 0.1601, + "3": 0.31548, + "4": 0.13221, + "5": 0.21062, + "6": 0.13175, + "7": 0.13948, + "8": 0.13253, + "9": 0.13332, + "10": 0.13374, + "11": 0.13408, + "12": 0.13166, + "13": 0.13216, + "14": 0.13242, + "15": 0.13287, + "16": 0.13083, + "17": 0.13222, + "18": 0.13109, + "19": 0.13227, + "20": 0.13185, + "21": 0.13322, + "22": 0.13244, + "23": 0.13117, + "24": 0.13111, + "25": 0.13055, + "26": 0.13118, + "27": 0.13026, + "28": 0.13082, + "29": 0.13115, + "30": 0.13195, + "31": 0.13225, + "32": 0.13125, + "33": 0.12998, + "34": 0.12996, + "35": 0.13024, + "36": 0.13128, + "37": 0.12958, + "38": 0.1303, + "39": 0.13166, + "40": 0.13058, + "41": 0.13178, + "42": 0.13042, + "43": 0.13185, + "44": 0.13057, + "45": 0.13021, + "46": 0.13066, + "47": 0.13074, + "48": 0.13024, + "49": 0.1312, + "50": 0.13056, + "51": 0.1315, + "52": 0.13065, + "53": 0.12957, + "54": 0.13022, + "55": 0.13003, + "56": 0.13074, + "57": 0.13086, + "58": 0.13085, + "59": 0.13122, + "60": 0.13094, + "61": 0.13054, + "62": 0.12934, + "63": 0.13189, + "64": 0.13037, + "65": 0.13106, + "66": 0.13119, + "67": 0.13091, + "68": 0.13142, + "69": 0.13076, + "70": 0.13182, + "71": 0.13066, + "72": 0.13164, + "73": 0.13144, + "74": 0.13143, + "75": 0.12994, + "76": 0.13106, + "77": 0.1312, + "78": 0.13129, + "79": 0.13102, + "80": 0.13101, + "81": 0.13117, + "82": 0.12994, + "83": 0.13181, + "84": 0.13005, + "85": 0.13069, + "86": 0.13072, + "87": 0.13113, + "88": 0.13057, + "89": 0.1316, + "90": 0.12979, + "91": 0.13056, + "92": 0.13048, + "93": 0.13105, + "94": 0.13021, + "95": 0.1302, + "96": 0.13023, + "97": 0.12981, + "98": 0.12888, + "99": 0.13229, + "100": 0.12801 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json new file mode 100644 index 00000000000..336272e4219 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79613, + "17": 10.76887, + "18": 10.81076, + "19": 10.7986, + "20": 10.69061, + "21": 10.67945, + "22": 10.52114, + "23": 10.70755, + "24": 10.57741, + "25": 10.52373, + "26": 10.59597, + "27": 10.58619, + "28": 10.56176, + "29": 10.56945, + "30": 10.34573, + "31": 10.10078, + "32": 10.45392, + "33": 10.4461, + "34": 10.20721, + "35": 10.262, + "36": 10.21312, + "37": 10.32468, + "38": 10.16831, + "39": 10.38374, + "40": 10.07231, + "41": 10.13763, + "42": 10.19765, + "43": 9.81155, + "44": 9.93311, + "45": 9.8104, + "46": 9.80854, + "47": 10.12558, + "48": 9.82105, + "49": 9.50764, + "50": 9.88382, + "51": 9.83549, + "52": 9.72516, + "53": 10.04799, + "54": 9.93011, + "55": 9.8636, + "56": 9.60217, + "57": 9.45187, + "58": 9.8078, + "59": 9.56783, + "60": 9.47966, + "61": 9.67984, + "62": 9.96754, + "63": 9.35113, + "64": 9.75623, + "65": 8.9318, + "66": 9.68107, + "67": 9.35956, + "68": 9.76948, + "69": 9.77492, + "70": 9.71182, + "71": 9.60632, + "72": 9.57129, + "73": 9.48392, + "74": 8.92911, + "75": 9.40028, + "76": 9.07194, + "77": 10.05252, + "78": 9.71494, + "79": 9.35747, + "80": 9.38946, + "81": 9.46791, + "82": 9.68508, + "83": 9.29588, + "84": 9.40522, + "85": 9.60163, + "86": 9.06713, + "87": 9.58402, + "88": 9.73304, + "89": 9.59526, + "90": 9.80555, + "91": 9.32604, + "92": 9.35323, + "93": 9.06915, + "94": 8.82268, + "95": 9.50858, + "96": 9.51584, + "97": 9.2976, + "98": 9.66184, + "99": 8.87662, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1728.0, + "16": 1773.0, + "17": 1805.0, + "18": 1702.0, + "19": 1803.0, + "20": 1627.0, + "21": 1822.0, + "22": 1748.0, + "23": 1938.0, + "24": 1642.0, + "25": 1696.0, + "26": 1760.0, + "27": 1809.0, + "28": 2025.0, + "29": 1900.0, + "30": 1902.0, + "31": 1645.0, + "32": 1876.0, + "33": 2105.0, + "34": 1881.0, + "35": 1913.0, + "36": 1864.0, + "37": 2322.0, + "38": 2194.0, + "39": 2318.0, + "40": 2010.0, + "41": 2358.0, + "42": 2155.0, + "43": 1980.0, + "44": 2105.0, + "45": 2082.0, + "46": 2221.0, + "47": 2537.0, + "48": 2367.0, + "49": 2190.0, + "50": 2352.0, + "51": 2441.0, + "52": 2482.0, + "53": 2916.0, + "54": 2550.0, + "55": 2347.0, + "56": 2765.0, + "57": 2116.0, + "58": 2968.0, + "59": 2810.0, + "60": 2384.0, + "61": 2912.0, + "62": 2554.0, + "63": 2364.0, + "64": 3035.0, + "65": 2648.0, + "66": 2979.0, + "67": 2741.0, + "68": 2799.0, + "69": 3071.0, + "70": 3098.0, + "71": 2950.0, + "72": 2342.0, + "73": 2829.0, + "74": 1840.0, + "75": 2426.0, + "76": 2941.0, + "77": 3245.0, + "78": 3272.0, + "79": 3066.0, + "80": 3221.0, + "81": 3565.0, + "82": 3162.0, + "83": 2876.0, + "84": 3180.0, + "85": 3410.0, + "86": 2778.0, + "87": 3752.0, + "88": 2995.0, + "89": 3264.0, + "90": 2940.0, + "91": 2791.0, + "92": 3118.0, + "93": 2634.0, + "94": 3464.0, + "95": 3344.0, + "96": 3499.0, + "97": 3122.0, + "98": 3568.0, + "99": 3272.0, + "100": 3476.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 731763200.0, + "2": 731763200.0, + "3": 731763200.0, + "4": 731763200.0, + "5": 731763200.0, + "6": 731763200.0, + "7": 731763200.0, + "8": 731763200.0, + "9": 731763200.0, + "10": 731763200.0, + "11": 731763200.0, + "12": 731763200.0, + "13": 731763200.0, + "14": 731763200.0, + "15": 731763200.0, + "16": 731763200.0, + "17": 731763200.0, + "18": 731763200.0, + "19": 731763200.0, + "20": 731763200.0, + "21": 731763200.0, + "22": 731763200.0, + "23": 731763200.0, + "24": 731763200.0, + "25": 731763200.0, + "26": 731763200.0, + "27": 731763200.0, + "28": 731763200.0, + "29": 731763200.0, + "30": 731763200.0, + "31": 731763200.0, + "32": 731763200.0, + "33": 731763200.0, + "34": 731763200.0, + "35": 731763200.0, + "36": 731763200.0, + "37": 731763200.0, + "38": 731763200.0, + "39": 731763200.0, + "40": 731763200.0, + "41": 731763200.0, + "42": 731763200.0, + "43": 731763200.0, + "44": 731763200.0, + "45": 731763200.0, + "46": 731763200.0, + "47": 731763200.0, + "48": 731763200.0, + "49": 731763200.0, + "50": 731763200.0, + "51": 731763200.0, + "52": 731763200.0, + "53": 731763200.0, + "54": 731763200.0, + "55": 731763200.0, + "56": 731763200.0, + "57": 731763200.0, + "58": 731763200.0, + "59": 731763200.0, + "60": 731763200.0, + "61": 731763200.0, + "62": 731763200.0, + "63": 731763200.0, + "64": 731763200.0, + "65": 731763200.0, + "66": 731763200.0, + "67": 731763200.0, + "68": 731763200.0, + "69": 731763200.0, + "70": 731763200.0, + "71": 731763200.0, + "72": 731763200.0, + "73": 731763200.0, + "74": 731763200.0, + "75": 731763200.0, + "76": 731763200.0, + "77": 731763200.0, + "78": 731763200.0, + "79": 731763200.0, + "80": 731763200.0, + "81": 731763200.0, + "82": 731763200.0, + "83": 731763200.0, + "84": 731763200.0, + "85": 731763200.0, + "86": 731763200.0, + "87": 731763200.0, + "88": 731763200.0, + "89": 731763200.0, + "90": 731763200.0, + "91": 731763200.0, + "92": 731763200.0, + "93": 731763200.0, + "94": 731763200.0, + "95": 731763200.0, + "96": 731763200.0, + "97": 731763200.0, + "98": 731763200.0, + "99": 731763200.0, + "100": 731763200.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2368927744.0, + "2": 2649590784.0, + "3": 2649590784.0, + "4": 2649590784.0, + "5": 2649590784.0, + "6": 2649590784.0, + "7": 2649590784.0, + "8": 2649590784.0, + "9": 2649590784.0, + "10": 2649590784.0, + "11": 2649590784.0, + "12": 2649590784.0, + "13": 2649590784.0, + "14": 2649590784.0, + "15": 2649590784.0, + "16": 2649590784.0, + "17": 2649590784.0, + "18": 2649590784.0, + "19": 2649590784.0, + "20": 2649590784.0, + "21": 2649590784.0, + "22": 2649590784.0, + "23": 2649590784.0, + "24": 2649590784.0, + "25": 2649590784.0, + "26": 2649590784.0, + "27": 2649590784.0, + "28": 2649590784.0, + "29": 2649590784.0, + "30": 2649590784.0, + "31": 2649590784.0, + "32": 2649590784.0, + "33": 2649590784.0, + "34": 2649590784.0, + "35": 2649590784.0, + "36": 2649590784.0, + "37": 2649590784.0, + "38": 2649590784.0, + "39": 2649590784.0, + "40": 2649590784.0, + "41": 2649590784.0, + "42": 2649590784.0, + "43": 2649590784.0, + "44": 2649590784.0, + "45": 2649590784.0, + "46": 2649590784.0, + "47": 2649590784.0, + "48": 2649590784.0, + "49": 2649590784.0, + "50": 2649590784.0, + "51": 2649590784.0, + "52": 2649590784.0, + "53": 2649590784.0, + "54": 2649590784.0, + "55": 2649590784.0, + "56": 2649590784.0, + "57": 2649590784.0, + "58": 2649590784.0, + "59": 2649590784.0, + "60": 2649590784.0, + "61": 2649590784.0, + "62": 2649590784.0, + "63": 2649590784.0, + "64": 2649590784.0, + "65": 2649590784.0, + "66": 2649590784.0, + "67": 2649590784.0, + "68": 2649590784.0, + "69": 2649590784.0, + "70": 2649590784.0, + "71": 2649590784.0, + "72": 2649590784.0, + "73": 2649590784.0, + "74": 2649590784.0, + "75": 2649590784.0, + "76": 2649590784.0, + "77": 2649590784.0, + "78": 2649590784.0, + "79": 2649590784.0, + "80": 2649590784.0, + "81": 2649590784.0, + "82": 2649590784.0, + "83": 2649590784.0, + "84": 2649590784.0, + "85": 2649590784.0, + "86": 2649590784.0, + "87": 2649590784.0, + "88": 2649590784.0, + "89": 2649590784.0, + "90": 2649590784.0, + "91": 2649590784.0, + "92": 2649590784.0, + "93": 2649590784.0, + "94": 2649590784.0, + "95": 2649590784.0, + "96": 2649590784.0, + "97": 2649590784.0, + "98": 2649590784.0, + "99": 2649590784.0, + "100": 2649590784.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 9.80619, + "2": 0.15632, + "3": 0.13276, + "4": 0.13108, + "5": 0.13211, + "6": 0.13413, + "7": 0.13422, + "8": 0.13375, + "9": 0.13411, + "10": 0.13309, + "11": 0.13357, + "12": 0.13347, + "13": 0.12915, + "14": 0.13018, + "15": 0.13025, + "16": 0.12753, + "17": 0.12441, + "18": 0.12502, + "19": 0.12557, + "20": 0.12467, + "21": 0.12438, + "22": 0.12486, + "23": 0.12591, + "24": 0.12519, + "25": 0.1238, + "26": 0.12519, + "27": 0.12417, + "28": 0.12497, + "29": 0.12468, + "30": 0.12796, + "31": 0.12718, + "32": 0.12629, + "33": 0.12682, + "34": 0.12877, + "35": 0.12684, + "36": 0.12612, + "37": 0.12643, + "38": 0.12701, + "39": 0.12729, + "40": 0.12702, + "41": 0.13876, + "42": 0.13281, + "43": 0.12918, + "44": 0.12822, + "45": 0.12516, + "46": 0.13017, + "47": 0.12894, + "48": 0.12956, + "49": 0.1285, + "50": 0.12784, + "51": 0.14972, + "52": 0.14056, + "53": 0.15351, + "54": 0.16383, + "55": 0.16094, + "56": 0.15475, + "57": 0.15074, + "58": 0.17471, + "59": 0.15262, + "60": 0.17912, + "61": 0.15182, + "62": 0.15731, + "63": 0.16109, + "64": 0.16116, + "65": 0.15248, + "66": 0.12939, + "67": 0.12927, + "68": 0.12873, + "69": 0.12982, + "70": 0.12904, + "71": 0.12845, + "72": 0.12939, + "73": 0.12993, + "74": 0.12822, + "75": 0.12923, + "76": 0.1286, + "77": 0.12871, + "78": 0.12825, + "79": 0.12821, + "80": 0.1283, + "81": 0.12931, + "82": 0.12824, + "83": 0.12975, + "84": 0.1281, + "85": 0.13353, + "86": 0.12896, + "87": 0.12873, + "88": 0.12803, + "89": 0.12965, + "90": 0.12863, + "91": 0.12795, + "92": 0.129, + "93": 0.12763, + "94": 0.12893, + "95": 0.12736, + "96": 0.23153, + "97": 0.12977, + "98": 0.12877, + "99": 0.12988, + "100": 0.12877 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..9fa574e32e0 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 520868352.0, + "2": 520868352.0, + "3": 520868352.0, + "4": 520868352.0, + "5": 520868352.0, + "6": 520868352.0, + "7": 520868352.0, + "8": 520868352.0, + "9": 520868352.0, + "10": 520868352.0, + "11": 520868352.0, + "12": 520868352.0, + "13": 520868352.0, + "14": 520868352.0, + "15": 520868352.0, + "16": 520868352.0, + "17": 520868352.0, + "18": 520868352.0, + "19": 520868352.0, + "20": 520868352.0, + "21": 520868352.0, + "22": 520868352.0, + "23": 520868352.0, + "24": 520868352.0, + "25": 520868352.0, + "26": 520868352.0, + "27": 520868352.0, + "28": 520868352.0, + "29": 520868352.0, + "30": 520868352.0, + "31": 520868352.0, + "32": 520868352.0, + "33": 520868352.0, + "34": 520868352.0, + "35": 520868352.0, + "36": 520868352.0, + "37": 520868352.0, + "38": 520868352.0, + "39": 520868352.0, + "40": 520868352.0, + "41": 520868352.0, + "42": 520868352.0, + "43": 520868352.0, + "44": 520868352.0, + "45": 520868352.0, + "46": 520868352.0, + "47": 520868352.0, + "48": 520868352.0, + "49": 520868352.0, + "50": 520868352.0, + "51": 520868352.0, + "52": 520868352.0, + "53": 520868352.0, + "54": 520868352.0, + "55": 520868352.0, + "56": 520868352.0, + "57": 520868352.0, + "58": 520868352.0, + "59": 520868352.0, + "60": 520868352.0, + "61": 520868352.0, + "62": 520868352.0, + "63": 520868352.0, + "64": 520868352.0, + "65": 520868352.0, + "66": 520868352.0, + "67": 520868352.0, + "68": 520868352.0, + "69": 520868352.0, + "70": 520868352.0, + "71": 520868352.0, + "72": 520868352.0, + "73": 520868352.0, + "74": 520868352.0, + "75": 520868352.0, + "76": 520868352.0, + "77": 520868352.0, + "78": 520868352.0, + "79": 520868352.0, + "80": 520868352.0, + "81": 520868352.0, + "82": 520868352.0, + "83": 520868352.0, + "84": 520868352.0, + "85": 520868352.0, + "86": 520868352.0, + "87": 520868352.0, + "88": 520868352.0, + "89": 520868352.0, + "90": 520868352.0, + "91": 520868352.0, + "92": 520868352.0, + "93": 520868352.0, + "94": 520868352.0, + "95": 520868352.0, + "96": 520868352.0, + "97": 520868352.0, + "98": 520868352.0, + "99": 520868352.0, + "100": 520868352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2297829376.0, + "2": 2439744512.0, + "3": 2439744512.0, + "4": 2439744512.0, + "5": 2439744512.0, + "6": 2439744512.0, + "7": 2439744512.0, + "8": 2439744512.0, + "9": 2439744512.0, + "10": 2439744512.0, + "11": 2439744512.0, + "12": 2439744512.0, + "13": 2439744512.0, + "14": 2439744512.0, + "15": 2439744512.0, + "16": 2439744512.0, + "17": 2439744512.0, + "18": 2439744512.0, + "19": 2439744512.0, + "20": 2439744512.0, + "21": 2439744512.0, + "22": 2439744512.0, + "23": 2439744512.0, + "24": 2439744512.0, + "25": 2439744512.0, + "26": 2439744512.0, + "27": 2439744512.0, + "28": 2439744512.0, + "29": 2439744512.0, + "30": 2439744512.0, + "31": 2439744512.0, + "32": 2439744512.0, + "33": 2439744512.0, + "34": 2439744512.0, + "35": 2439744512.0, + "36": 2439744512.0, + "37": 2439744512.0, + "38": 2439744512.0, + "39": 2439744512.0, + "40": 2439744512.0, + "41": 2439744512.0, + "42": 2439744512.0, + "43": 2439744512.0, + "44": 2439744512.0, + "45": 2439744512.0, + "46": 2439744512.0, + "47": 2439744512.0, + "48": 2439744512.0, + "49": 2439744512.0, + "50": 2439744512.0, + "51": 2439744512.0, + "52": 2439744512.0, + "53": 2439744512.0, + "54": 2439744512.0, + "55": 2439744512.0, + "56": 2439744512.0, + "57": 2439744512.0, + "58": 2439744512.0, + "59": 2439744512.0, + "60": 2439744512.0, + "61": 2439744512.0, + "62": 2439744512.0, + "63": 2439744512.0, + "64": 2439744512.0, + "65": 2439744512.0, + "66": 2439744512.0, + "67": 2439744512.0, + "68": 2439744512.0, + "69": 2439744512.0, + "70": 2439744512.0, + "71": 2439744512.0, + "72": 2439744512.0, + "73": 2439744512.0, + "74": 2439744512.0, + "75": 2439744512.0, + "76": 2439744512.0, + "77": 2439744512.0, + "78": 2439744512.0, + "79": 2439744512.0, + "80": 2439744512.0, + "81": 2439744512.0, + "82": 2439744512.0, + "83": 2439744512.0, + "84": 2439744512.0, + "85": 2439744512.0, + "86": 2439744512.0, + "87": 2439744512.0, + "88": 2439744512.0, + "89": 2439744512.0, + "90": 2439744512.0, + "91": 2439744512.0, + "92": 2439744512.0, + "93": 2439744512.0, + "94": 2439744512.0, + "95": 2439744512.0, + "96": 2439744512.0, + "97": 2439744512.0, + "98": 2439744512.0, + "99": 2439744512.0, + "100": 2439744512.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79372, + "2": 0.17087, + "3": 0.14208, + "4": 0.13928, + "5": 0.14226, + "6": 0.13749, + "7": 0.14244, + "8": 0.13559, + "9": 0.13672, + "10": 0.13944, + "11": 0.13837, + "12": 0.14131, + "13": 0.14226, + "14": 0.13614, + "15": 0.13758, + "16": 0.13716, + "17": 0.13878, + "18": 0.13767, + "19": 0.14136, + "20": 0.13828, + "21": 0.13741, + "22": 0.13887, + "23": 0.13728, + "24": 0.13706, + "25": 0.13843, + "26": 0.13733, + "27": 0.1376, + "28": 0.16069, + "29": 0.13762, + "30": 0.13726, + "31": 0.13872, + "32": 0.13889, + "33": 0.13974, + "34": 0.13756, + "35": 0.13783, + "36": 0.13756, + "37": 0.13462, + "38": 0.13749, + "39": 0.13717, + "40": 0.13587, + "41": 0.13851, + "42": 0.1368, + "43": 0.13695, + "44": 0.13444, + "45": 0.13462, + "46": 0.13663, + "47": 0.13487, + "48": 0.13518, + "49": 0.13573, + "50": 0.13547, + "51": 0.13826, + "52": 0.13644, + "53": 0.13455, + "54": 0.13513, + "55": 0.13449, + "56": 0.13421, + "57": 0.13293, + "58": 0.13443, + "59": 0.13476, + "60": 0.13426, + "61": 0.13363, + "62": 0.13337, + "63": 0.13515, + "64": 0.13296, + "65": 0.1344, + "66": 0.13486, + "67": 0.13512, + "68": 0.13329, + "69": 0.13364, + "70": 0.13337, + "71": 0.13314, + "72": 0.13535, + "73": 0.13447, + "74": 0.13336, + "75": 0.13237, + "76": 0.13271, + "77": 0.13444, + "78": 0.13323, + "79": 0.13343, + "80": 0.13438, + "81": 0.13288, + "82": 0.13278, + "83": 0.13391, + "84": 0.13419, + "85": 0.13429, + "86": 0.13382, + "87": 0.13502, + "88": 0.13402, + "89": 0.13495, + "90": 0.13317, + "91": 0.13496, + "92": 0.13384, + "93": 0.1344, + "94": 0.13257, + "95": 0.13384, + "96": 0.13316, + "97": 0.13413, + "98": 0.13402, + "99": 0.13376, + "100": 0.13303 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..88353778184 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81548, + "2": 10.8208, + "3": 10.81272, + "4": 10.78885, + "5": 10.85169, + "6": 10.87038, + "7": 10.83098, + "8": 10.8343, + "9": 10.83996, + "10": 10.78686, + "11": 10.88025, + "12": 10.85974, + "13": 10.86596, + "14": 10.8782, + "15": 10.79466, + "16": 10.79611, + "17": 10.76888, + "18": 10.81074, + "19": 10.79865, + "20": 10.69058, + "21": 10.67945, + "22": 10.52113, + "23": 10.70755, + "24": 10.5774, + "25": 10.52375, + "26": 10.59601, + "27": 10.58621, + "28": 10.56178, + "29": 10.56944, + "30": 10.34575, + "31": 10.10079, + "32": 10.45392, + "33": 10.4461, + "34": 10.20725, + "35": 10.26198, + "36": 10.21312, + "37": 10.32467, + "38": 10.16829, + "39": 10.38371, + "40": 10.07233, + "41": 10.13762, + "42": 10.19766, + "43": 9.81156, + "44": 9.93307, + "45": 9.81043, + "46": 9.80854, + "47": 10.12558, + "48": 9.82104, + "49": 9.50761, + "50": 9.8838, + "51": 9.83548, + "52": 9.72518, + "53": 10.04799, + "54": 9.93007, + "55": 9.86362, + "56": 9.60218, + "57": 9.45185, + "58": 9.80781, + "59": 9.56786, + "60": 9.47966, + "61": 9.67985, + "62": 9.9675, + "63": 9.35111, + "64": 9.75622, + "65": 8.93178, + "66": 9.68108, + "67": 9.35959, + "68": 9.76948, + "69": 9.77494, + "70": 9.71179, + "71": 9.60631, + "72": 9.57134, + "73": 9.48393, + "74": 8.92913, + "75": 9.4003, + "76": 9.07189, + "77": 10.05248, + "78": 9.71492, + "79": 9.35744, + "80": 9.38946, + "81": 9.46798, + "82": 9.68509, + "83": 9.29591, + "84": 9.40521, + "85": 9.60161, + "86": 9.06713, + "87": 9.58406, + "88": 9.73301, + "89": 9.59528, + "90": 9.80559, + "91": 9.32603, + "92": 9.3532, + "93": 9.06916, + "94": 8.82266, + "95": 9.50858, + "96": 9.51587, + "97": 9.29763, + "98": 9.66187, + "99": 8.87661, + "100": 9.39222 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1518.0, + "2": 1697.0, + "3": 1678.0, + "4": 1709.0, + "5": 1873.0, + "6": 1786.0, + "7": 1813.0, + "8": 1519.0, + "9": 1684.0, + "10": 1417.0, + "11": 1943.0, + "12": 1723.0, + "13": 1939.0, + "14": 1759.0, + "15": 1772.0, + "16": 1750.0, + "17": 1812.0, + "18": 1646.0, + "19": 1818.0, + "20": 1655.0, + "21": 1879.0, + "22": 1759.0, + "23": 1944.0, + "24": 1650.0, + "25": 1708.0, + "26": 1750.0, + "27": 1815.0, + "28": 2091.0, + "29": 1958.0, + "30": 1934.0, + "31": 1588.0, + "32": 1891.0, + "33": 2099.0, + "34": 1765.0, + "35": 1894.0, + "36": 1947.0, + "37": 2279.0, + "38": 2134.0, + "39": 2350.0, + "40": 2091.0, + "41": 2318.0, + "42": 2137.0, + "43": 1849.0, + "44": 2109.0, + "45": 2020.0, + "46": 2263.0, + "47": 2593.0, + "48": 2323.0, + "49": 2177.0, + "50": 2363.0, + "51": 2554.0, + "52": 2619.0, + "53": 2863.0, + "54": 2688.0, + "55": 2406.0, + "56": 2649.0, + "57": 2175.0, + "58": 2856.0, + "59": 2775.0, + "60": 2307.0, + "61": 2914.0, + "62": 2644.0, + "63": 2362.0, + "64": 2946.0, + "65": 2578.0, + "66": 3122.0, + "67": 2697.0, + "68": 2687.0, + "69": 2956.0, + "70": 3157.0, + "71": 3028.0, + "72": 2294.0, + "73": 2876.0, + "74": 1887.0, + "75": 2523.0, + "76": 2937.0, + "77": 3162.0, + "78": 3318.0, + "79": 3074.0, + "80": 3213.0, + "81": 3664.0, + "82": 3238.0, + "83": 2838.0, + "84": 3251.0, + "85": 3275.0, + "86": 2748.0, + "87": 3758.0, + "88": 3023.0, + "89": 3267.0, + "90": 3085.0, + "91": 2812.0, + "92": 3116.0, + "93": 2665.0, + "94": 3380.0, + "95": 3236.0, + "96": 3462.0, + "97": 3002.0, + "98": 3545.0, + "99": 3265.0, + "100": 3458.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.62871, + "2": 0.19397, + "3": 0.15653, + "4": 0.14786, + "5": 0.14498, + "6": 0.13762, + "7": 0.13635, + "8": 0.13679, + "9": 0.13676, + "10": 0.13607, + "11": 0.30376, + "12": 0.14098, + "13": 0.13882, + "14": 0.13845, + "15": 0.13866, + "16": 0.13713, + "17": 0.13779, + "18": 0.13753, + "19": 0.13863, + "20": 0.13947, + "21": 0.30215, + "22": 0.13844, + "23": 0.13918, + "24": 0.13928, + "25": 0.13639, + "26": 0.13796, + "27": 0.13788, + "28": 0.13695, + "29": 0.13855, + "30": 0.13835, + "31": 0.30176, + "32": 0.13753, + "33": 0.13687, + "34": 0.13683, + "35": 0.13834, + "36": 0.13723, + "37": 0.1369, + "38": 0.13761, + "39": 0.13741, + "40": 0.13774, + "41": 0.30109, + "42": 0.13882, + "43": 0.13859, + "44": 0.13772, + "45": 0.13734, + "46": 0.13838, + "47": 0.13674, + "48": 0.13706, + "49": 0.13705, + "50": 0.13748, + "51": 0.29954, + "52": 0.13753, + "53": 0.13687, + "54": 0.13685, + "55": 0.13705, + "56": 0.13843, + "57": 0.13672, + "58": 0.1374, + "59": 0.1378, + "60": 0.13768, + "61": 0.28492, + "62": 0.1365, + "63": 0.13786, + "64": 0.13726, + "65": 0.13752, + "66": 0.13775, + "67": 0.13727, + "68": 0.13779, + "69": 0.13722, + "70": 0.13736, + "71": 0.28581, + "72": 0.14137, + "73": 0.1401, + "74": 0.1394, + "75": 0.13929, + "76": 0.13873, + "77": 0.13914, + "78": 0.1384, + "79": 0.1395, + "80": 0.13872, + "81": 0.28715, + "82": 0.13997, + "83": 0.14057, + "84": 0.13909, + "85": 0.13939, + "86": 0.13914, + "87": 0.14171, + "88": 0.13951, + "89": 0.14196, + "90": 0.13998, + "91": 0.28348, + "92": 0.13965, + "93": 0.14004, + "94": 0.14077, + "95": 0.13874, + "96": 0.13934, + "97": 0.14098, + "98": 0.1387, + "99": 0.13948, + "100": 0.1392 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json new file mode 100644 index 00000000000..0323a8bbccd --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93693, + "2": 10.92078, + "3": 10.92311, + "4": 10.908, + "5": 10.93133, + "6": 10.93476, + "7": 10.92722, + "8": 10.91987, + "9": 10.93801, + "10": 10.91377, + "11": 10.94194, + "12": 10.93222, + "13": 10.92553, + "14": 10.94301, + "15": 10.86288, + "16": 10.87945, + "17": 10.86904, + "18": 10.88527, + "19": 10.86912, + "20": 10.77336, + "21": 10.75658, + "22": 10.62975, + "23": 10.76834, + "24": 10.65017, + "25": 10.60362, + "26": 10.66269, + "27": 10.66488, + "28": 10.60369, + "29": 10.64791, + "30": 10.40493, + "31": 10.16869, + "32": 10.5139, + "33": 10.50484, + "34": 10.27115, + "35": 10.31433, + "36": 10.27029, + "37": 10.38626, + "38": 10.23175, + "39": 10.45527, + "40": 10.12357, + "41": 10.19645, + "42": 10.25476, + "43": 9.86653, + "44": 9.99586, + "45": 9.87497, + "46": 9.86189, + "47": 10.19545, + "48": 9.87912, + "49": 9.56741, + "50": 9.94519, + "51": 9.89774, + "52": 9.78773, + "53": 10.12739, + "54": 9.98265, + "55": 9.90115, + "56": 9.66568, + "57": 9.49996, + "58": 9.87635, + "59": 9.61861, + "60": 9.55292, + "61": 9.71598, + "62": 10.03451, + "63": 9.41294, + "64": 9.81949, + "65": 8.96909, + "66": 9.7478, + "67": 9.39393, + "68": 9.82085, + "69": 9.82417, + "70": 9.77915, + "71": 9.64756, + "72": 9.59555, + "73": 9.53704, + "74": 8.96643, + "75": 9.44556, + "76": 9.09922, + "77": 10.10009, + "78": 9.73576, + "79": 9.38721, + "80": 9.41961, + "81": 9.50968, + "82": 9.72303, + "83": 9.34684, + "84": 9.44855, + "85": 9.64312, + "86": 9.07818, + "87": 9.61561, + "88": 9.79072, + "89": 9.61878, + "90": 9.85929, + "91": 9.35347, + "92": 9.38799, + "93": 9.07947, + "94": 8.82329, + "95": 9.52141, + "96": 9.54598, + "97": 9.34241, + "98": 9.70512, + "99": 8.8917, + "100": 9.43443 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22727688.0, + "2": 22924916.0, + "3": 22596752.0, + "4": 23219574.0, + "5": 22715594.0, + "6": 23021444.0, + "7": 22771554.0, + "8": 22926856.0, + "9": 22842096.0, + "10": 22918196.0, + "11": 22500760.0, + "12": 22460572.0, + "13": 22917018.0, + "14": 22389964.0, + "15": 22820720.0, + "16": 22830308.0, + "17": 22819122.0, + "18": 22582876.0, + "19": 22617912.0, + "20": 22693480.0, + "21": 22740412.0, + "22": 22799838.0, + "23": 22539332.0, + "24": 22770580.0, + "25": 22818910.0, + "26": 22547838.0, + "27": 22468232.0, + "28": 22452766.0, + "29": 22528906.0, + "30": 22631718.0, + "31": 22955520.0, + "32": 22584888.0, + "33": 22558558.0, + "34": 22836286.0, + "35": 22788956.0, + "36": 22589940.0, + "37": 22496912.0, + "38": 22897132.0, + "39": 22801712.0, + "40": 22657514.0, + "41": 22659596.0, + "42": 22667300.0, + "43": 22976308.0, + "44": 22745816.0, + "45": 22675136.0, + "46": 22884540.0, + "47": 22633726.0, + "48": 22928020.0, + "49": 22727656.0, + "50": 22905162.0, + "51": 22791660.0, + "52": 22748630.0, + "53": 22925420.0, + "54": 22839176.0, + "55": 22518792.0, + "56": 22877648.0, + "57": 23113592.0, + "58": 22845136.0, + "59": 22715486.0, + "60": 22743726.0, + "61": 22724104.0, + "62": 22673746.0, + "63": 22846740.0, + "64": 22823862.0, + "65": 23061360.0, + "66": 22729628.0, + "67": 22907694.0, + "68": 22609888.0, + "69": 22584610.0, + "70": 22829190.0, + "71": 22749038.0, + "72": 22655052.0, + "73": 22739796.0, + "74": 23047606.0, + "75": 23054054.0, + "76": 22901052.0, + "77": 22271520.0, + "78": 22788892.0, + "79": 22743418.0, + "80": 22706694.0, + "81": 22890786.0, + "82": 22777316.0, + "83": 22839636.0, + "84": 23010088.0, + "85": 22712948.0, + "86": 23103380.0, + "87": 22735258.0, + "88": 22636824.0, + "89": 22498828.0, + "90": 22972694.0, + "91": 22767128.0, + "92": 22808892.0, + "93": 22659368.0, + "94": 22911112.0, + "95": 23047740.0, + "96": 22828678.0, + "97": 22608174.0, + "98": 22762982.0, + "99": 22905800.0, + "100": 23015792.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 519038464.0, + "2": 519038464.0, + "3": 519038464.0, + "4": 519038464.0, + "5": 519038464.0, + "6": 519038464.0, + "7": 519038464.0, + "8": 519038464.0, + "9": 519038464.0, + "10": 519038464.0, + "11": 519038464.0, + "12": 519038464.0, + "13": 519038464.0, + "14": 519038464.0, + "15": 519038464.0, + "16": 519038464.0, + "17": 519038464.0, + "18": 519038464.0, + "19": 519038464.0, + "20": 519038464.0, + "21": 519038464.0, + "22": 519038464.0, + "23": 519038464.0, + "24": 519038464.0, + "25": 519038464.0, + "26": 519038464.0, + "27": 519038464.0, + "28": 519038464.0, + "29": 519038464.0, + "30": 519038464.0, + "31": 519038464.0, + "32": 519038464.0, + "33": 519038464.0, + "34": 519038464.0, + "35": 519038464.0, + "36": 519038464.0, + "37": 519038464.0, + "38": 519038464.0, + "39": 519038464.0, + "40": 519038464.0, + "41": 519038464.0, + "42": 519038464.0, + "43": 519038464.0, + "44": 519038464.0, + "45": 519038464.0, + "46": 519038464.0, + "47": 519038464.0, + "48": 519038464.0, + "49": 519038464.0, + "50": 519038464.0, + "51": 519038464.0, + "52": 519038464.0, + "53": 519038464.0, + "54": 519038464.0, + "55": 519038464.0, + "56": 519038464.0, + "57": 519038464.0, + "58": 519038464.0, + "59": 519038464.0, + "60": 519038464.0, + "61": 519038464.0, + "62": 519038464.0, + "63": 519038464.0, + "64": 519038464.0, + "65": 519038464.0, + "66": 519038464.0, + "67": 519038464.0, + "68": 519038464.0, + "69": 519038464.0, + "70": 519038464.0, + "71": 519038464.0, + "72": 519038464.0, + "73": 519038464.0, + "74": 519038464.0, + "75": 519038464.0, + "76": 519038464.0, + "77": 519038464.0, + "78": 519038464.0, + "79": 519038464.0, + "80": 519038464.0, + "81": 519038464.0, + "82": 519038464.0, + "83": 519038464.0, + "84": 519038464.0, + "85": 519038464.0, + "86": 519038464.0, + "87": 519038464.0, + "88": 519038464.0, + "89": 519038464.0, + "90": 519038464.0, + "91": 519038464.0, + "92": 519038464.0, + "93": 519038464.0, + "94": 519038464.0, + "95": 519038464.0, + "96": 519038464.0, + "97": 519038464.0, + "98": 519038464.0, + "99": 519038464.0, + "100": 519038464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2296009728.0, + "2": 2436866048.0, + "3": 2436866048.0, + "4": 2436866048.0, + "5": 2436866048.0, + "6": 2436866048.0, + "7": 2436866048.0, + "8": 2436866048.0, + "9": 2436866048.0, + "10": 2436866048.0, + "11": 2436866048.0, + "12": 2436866048.0, + "13": 2436866048.0, + "14": 2436866048.0, + "15": 2436866048.0, + "16": 2436866048.0, + "17": 2436866048.0, + "18": 2436866048.0, + "19": 2436866048.0, + "20": 2436866048.0, + "21": 2436866048.0, + "22": 2436866048.0, + "23": 2436866048.0, + "24": 2436866048.0, + "25": 2436866048.0, + "26": 2436866048.0, + "27": 2436866048.0, + "28": 2436866048.0, + "29": 2436866048.0, + "30": 2436866048.0, + "31": 2436866048.0, + "32": 2436866048.0, + "33": 2436866048.0, + "34": 2436866048.0, + "35": 2436866048.0, + "36": 2436866048.0, + "37": 2436866048.0, + "38": 2436866048.0, + "39": 2436866048.0, + "40": 2436866048.0, + "41": 2436866048.0, + "42": 2436866048.0, + "43": 2436866048.0, + "44": 2436866048.0, + "45": 2436866048.0, + "46": 2436866048.0, + "47": 2436866048.0, + "48": 2436866048.0, + "49": 2436866048.0, + "50": 2436866048.0, + "51": 2436866048.0, + "52": 2436866048.0, + "53": 2436866048.0, + "54": 2436866048.0, + "55": 2436866048.0, + "56": 2436866048.0, + "57": 2436866048.0, + "58": 2436866048.0, + "59": 2436866048.0, + "60": 2436866048.0, + "61": 2436866048.0, + "62": 2436866048.0, + "63": 2436866048.0, + "64": 2436866048.0, + "65": 2436866048.0, + "66": 2436866048.0, + "67": 2436866048.0, + "68": 2436866048.0, + "69": 2436866048.0, + "70": 2436866048.0, + "71": 2436866048.0, + "72": 2436866048.0, + "73": 2436866048.0, + "74": 2436866048.0, + "75": 2436866048.0, + "76": 2436866048.0, + "77": 2436866048.0, + "78": 2436866048.0, + "79": 2436866048.0, + "80": 2436866048.0, + "81": 2436866048.0, + "82": 2436866048.0, + "83": 2436866048.0, + "84": 2436866048.0, + "85": 2436866048.0, + "86": 2436866048.0, + "87": 2436866048.0, + "88": 2436866048.0, + "89": 2436866048.0, + "90": 2436866048.0, + "91": 2436866048.0, + "92": 2436866048.0, + "93": 2436866048.0, + "94": 2436866048.0, + "95": 2436866048.0, + "96": 2436866048.0, + "97": 2436866048.0, + "98": 2436866048.0, + "99": 2436866048.0, + "100": 2436866048.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.24766, + "2": 0.16317, + "3": 0.13869, + "4": 0.13535, + "5": 0.13147, + "6": 0.12936, + "7": 0.12838, + "8": 0.12754, + "9": 0.12694, + "10": 0.12785, + "11": 0.12885, + "12": 0.12708, + "13": 0.12786, + "14": 0.12845, + "15": 0.12831, + "16": 0.12834, + "17": 0.12782, + "18": 0.1282, + "19": 0.12831, + "20": 0.12722, + "21": 0.12738, + "22": 0.12758, + "23": 0.1286, + "24": 0.12783, + "25": 0.12797, + "26": 0.12713, + "27": 0.12871, + "28": 0.12669, + "29": 0.12782, + "30": 0.12767, + "31": 0.12765, + "32": 0.12768, + "33": 0.1273, + "34": 0.12735, + "35": 0.12732, + "36": 0.12747, + "37": 0.12741, + "38": 0.12748, + "39": 0.12732, + "40": 0.12783, + "41": 0.12596, + "42": 0.12509, + "43": 0.1264, + "44": 0.12601, + "45": 0.1259, + "46": 0.12793, + "47": 0.12759, + "48": 0.12775, + "49": 0.12762, + "50": 0.12724, + "51": 0.13584, + "52": 0.13364, + "53": 0.13184, + "54": 0.13408, + "55": 0.13253, + "56": 0.13188, + "57": 0.13282, + "58": 0.13216, + "59": 0.13222, + "60": 0.13328, + "61": 0.13159, + "62": 0.13191, + "63": 0.13164, + "64": 0.13107, + "65": 0.13189, + "66": 0.13097, + "67": 0.13152, + "68": 0.13173, + "69": 0.13137, + "70": 0.13125, + "71": 0.13096, + "72": 0.13128, + "73": 0.13136, + "74": 0.13099, + "75": 0.13041, + "76": 0.13067, + "77": 0.13057, + "78": 0.13113, + "79": 0.13152, + "80": 0.132, + "81": 0.13114, + "82": 0.13116, + "83": 0.13055, + "84": 0.13034, + "85": 0.13129, + "86": 0.13442, + "87": 0.13383, + "88": 0.13069, + "89": 0.13209, + "90": 0.13095, + "91": 0.13093, + "92": 0.13082, + "93": 0.13126, + "94": 0.13041, + "95": 0.13032, + "96": 0.13051, + "97": 0.13551, + "98": 0.13482, + "99": 0.13442, + "100": 0.13483 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..96a26c7439a --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80831, + "2": 10.81091, + "3": 10.82881, + "4": 10.82888, + "5": 10.8518, + "6": 10.84129, + "7": 10.839, + "8": 10.80345, + "9": 10.87875, + "10": 10.88064, + "11": 10.87605, + "12": 10.82786, + "13": 10.84179, + "14": 10.81686, + "15": 10.80637, + "16": 10.79763, + "17": 10.76624, + "18": 10.78069, + "19": 10.75188, + "20": 10.63371, + "21": 10.68119, + "22": 10.63819, + "23": 10.75544, + "24": 10.61129, + "25": 10.47478, + "26": 10.59702, + "27": 10.53977, + "28": 10.45162, + "29": 10.39421, + "30": 10.39348, + "31": 10.49397, + "32": 10.32158, + "33": 10.27676, + "34": 10.44629, + "35": 9.96856, + "36": 10.11335, + "37": 10.02306, + "38": 10.37579, + "39": 9.78845, + "40": 10.10124, + "41": 10.12323, + "42": 10.02969, + "43": 10.19869, + "44": 10.05528, + "45": 9.68289, + "46": 9.98143, + "47": 9.92628, + "48": 9.66986, + "49": 9.9198, + "50": 9.9226, + "51": 9.79344, + "52": 9.32777, + "53": 9.65056, + "54": 9.8608, + "55": 9.98167, + "56": 9.81701, + "57": 9.74358, + "58": 9.83, + "59": 9.32869, + "60": 9.35336, + "61": 9.45108, + "62": 10.19054, + "63": 9.35643, + "64": 9.62865, + "65": 9.70227, + "66": 9.52515, + "67": 9.6623, + "68": 9.58775, + "69": 9.38554, + "70": 9.73858, + "71": 9.87574, + "72": 9.69164, + "73": 9.39216, + "74": 9.43911, + "75": 8.95557, + "76": 9.56378, + "77": 9.6144, + "78": 9.39189, + "79": 9.52909, + "80": 9.31414, + "81": 9.70167, + "82": 9.90384, + "83": 9.31614, + "84": 9.47152, + "85": 8.97772, + "86": 9.66417, + "87": 9.43151, + "88": 9.58677, + "89": 9.52297, + "90": 9.55832, + "91": 9.62794, + "92": 9.14014, + "93": 9.42404, + "94": 9.54555, + "95": 9.13499, + "96": 8.75216, + "97": 9.58103, + "98": 9.79018, + "99": 9.37904, + "100": 9.21212 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1004.0, + "2": 1243.0, + "3": 1236.0, + "4": 1235.0, + "5": 1179.0, + "6": 1319.0, + "7": 1173.0, + "8": 1020.0, + "9": 1205.0, + "10": 1270.0, + "11": 1163.0, + "12": 1276.0, + "13": 1398.0, + "14": 1096.0, + "15": 1148.0, + "16": 1104.0, + "17": 1141.0, + "18": 1312.0, + "19": 1200.0, + "20": 1094.0, + "21": 1197.0, + "22": 1286.0, + "23": 1353.0, + "24": 1321.0, + "25": 1053.0, + "26": 1219.0, + "27": 1300.0, + "28": 1428.0, + "29": 1300.0, + "30": 1550.0, + "31": 1500.0, + "32": 1450.0, + "33": 1597.0, + "34": 1487.0, + "35": 1196.0, + "36": 1333.0, + "37": 1457.0, + "38": 1618.0, + "39": 1321.0, + "40": 1467.0, + "41": 1743.0, + "42": 1583.0, + "43": 1726.0, + "44": 1687.0, + "45": 1607.0, + "46": 1887.0, + "47": 1694.0, + "48": 1589.0, + "49": 1754.0, + "50": 1681.0, + "51": 1792.0, + "52": 1765.0, + "53": 1874.0, + "54": 1854.0, + "55": 1816.0, + "56": 1894.0, + "57": 1855.0, + "58": 1733.0, + "59": 1425.0, + "60": 1916.0, + "61": 2236.0, + "62": 2138.0, + "63": 2016.0, + "64": 2084.0, + "65": 2473.0, + "66": 2130.0, + "67": 2275.0, + "68": 2207.0, + "69": 2199.0, + "70": 2462.0, + "71": 2390.0, + "72": 2501.0, + "73": 1957.0, + "74": 2120.0, + "75": 2095.0, + "76": 2325.0, + "77": 2395.0, + "78": 2502.0, + "79": 2700.0, + "80": 1996.0, + "81": 2404.0, + "82": 2501.0, + "83": 2493.0, + "84": 2074.0, + "85": 2167.0, + "86": 2273.0, + "87": 2540.0, + "88": 2295.0, + "89": 2583.0, + "90": 2139.0, + "91": 2390.0, + "92": 1929.0, + "93": 2179.0, + "94": 2522.0, + "95": 2468.0, + "96": 2219.0, + "97": 2145.0, + "98": 2343.0, + "99": 2330.0, + "100": 1971.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0, + "51": 730321408.0, + "52": 730321408.0, + "53": 730321408.0, + "54": 730321408.0, + "55": 730321408.0, + "56": 730321408.0, + "57": 730321408.0, + "58": 730321408.0, + "59": 730321408.0, + "60": 730321408.0, + "61": 730321408.0, + "62": 730321408.0, + "63": 730321408.0, + "64": 730321408.0, + "65": 730321408.0, + "66": 730321408.0, + "67": 730321408.0, + "68": 730321408.0, + "69": 730321408.0, + "70": 730321408.0, + "71": 730321408.0, + "72": 730321408.0, + "73": 730321408.0, + "74": 730321408.0, + "75": 730321408.0, + "76": 730321408.0, + "77": 730321408.0, + "78": 730321408.0, + "79": 730321408.0, + "80": 730321408.0, + "81": 730321408.0, + "82": 730321408.0, + "83": 730321408.0, + "84": 730321408.0, + "85": 730321408.0, + "86": 730321408.0, + "87": 730321408.0, + "88": 730321408.0, + "89": 730321408.0, + "90": 730321408.0, + "91": 730321408.0, + "92": 730321408.0, + "93": 730321408.0, + "94": 730321408.0, + "95": 730321408.0, + "96": 730321408.0, + "97": 730321408.0, + "98": 730321408.0, + "99": 730321408.0, + "100": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2513633792.0, + "2": 2795345408.0, + "3": 2795345408.0, + "4": 2795345408.0, + "5": 2795345408.0, + "6": 2795345408.0, + "7": 2795345408.0, + "8": 2795345408.0, + "9": 2795345408.0, + "10": 2795345408.0, + "11": 2795345408.0, + "12": 2795345408.0, + "13": 2795345408.0, + "14": 2795345408.0, + "15": 2795345408.0, + "16": 2795345408.0, + "17": 2795345408.0, + "18": 2795345408.0, + "19": 2795345408.0, + "20": 2795345408.0, + "21": 2795345408.0, + "22": 2795345408.0, + "23": 2795345408.0, + "24": 2795345408.0, + "25": 2795345408.0, + "26": 2795345408.0, + "27": 2795345408.0, + "28": 2795345408.0, + "29": 2795345408.0, + "30": 2795345408.0, + "31": 2795345408.0, + "32": 2795345408.0, + "33": 2795345408.0, + "34": 2795345408.0, + "35": 2795345408.0, + "36": 2795345408.0, + "37": 2795345408.0, + "38": 2795345408.0, + "39": 2795345408.0, + "40": 2795345408.0, + "41": 2795345408.0, + "42": 2795345408.0, + "43": 2795345408.0, + "44": 2795345408.0, + "45": 2795345408.0, + "46": 2795345408.0, + "47": 2795345408.0, + "48": 2795345408.0, + "49": 2795345408.0, + "50": 2795345408.0, + "51": 2795345408.0, + "52": 2795345408.0, + "53": 2795345408.0, + "54": 2795345408.0, + "55": 2795345408.0, + "56": 2795345408.0, + "57": 2795345408.0, + "58": 2795345408.0, + "59": 2795345408.0, + "60": 2795345408.0, + "61": 2795345408.0, + "62": 2795345408.0, + "63": 2795345408.0, + "64": 2795345408.0, + "65": 2795345408.0, + "66": 2795345408.0, + "67": 2795345408.0, + "68": 2795345408.0, + "69": 2795345408.0, + "70": 2795345408.0, + "71": 2795345408.0, + "72": 2795345408.0, + "73": 2795345408.0, + "74": 2795345408.0, + "75": 2795345408.0, + "76": 2795345408.0, + "77": 2795345408.0, + "78": 2795345408.0, + "79": 2795345408.0, + "80": 2795345408.0, + "81": 2795345408.0, + "82": 2795345408.0, + "83": 2795345408.0, + "84": 2795345408.0, + "85": 2795345408.0, + "86": 2795345408.0, + "87": 2795345408.0, + "88": 2795345408.0, + "89": 2795345408.0, + "90": 2795345408.0, + "91": 2795345408.0, + "92": 2795345408.0, + "93": 2795345408.0, + "94": 2795345408.0, + "95": 2795345408.0, + "96": 2795345408.0, + "97": 2795345408.0, + "98": 2795345408.0, + "99": 2795345408.0, + "100": 2795345408.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.87224, + "2": 0.27072, + "3": 0.16155, + "4": 0.16114, + "5": 0.16274, + "6": 0.16196, + "7": 0.16051, + "8": 0.16068, + "9": 0.15822, + "10": 0.15542, + "11": 0.15273, + "12": 0.15488, + "13": 0.15899, + "14": 0.15628, + "15": 0.15591, + "16": 0.15649, + "17": 0.15897, + "18": 0.15947, + "19": 0.15901, + "20": 0.16054, + "21": 0.16154, + "22": 0.15643, + "23": 0.1582, + "24": 0.15756, + "25": 0.15696, + "26": 0.15573, + "27": 0.16642, + "28": 0.15596, + "29": 0.15961, + "30": 0.15463, + "31": 0.15904, + "32": 0.15603, + "33": 0.15861, + "34": 0.15702, + "35": 0.15376, + "36": 0.15598, + "37": 0.15465, + "38": 0.15958, + "39": 0.15572, + "40": 0.15838, + "41": 0.15741, + "42": 0.15879, + "43": 0.15617, + "44": 0.15742, + "45": 0.15803, + "46": 0.15538, + "47": 0.15744, + "48": 0.15794, + "49": 0.15846, + "50": 0.15778, + "51": 0.15546, + "52": 0.15315, + "53": 0.15443, + "54": 0.15307, + "55": 0.15185, + "56": 0.15253, + "57": 0.15496, + "58": 0.1563, + "59": 0.15244, + "60": 0.15606, + "61": 0.15852, + "62": 0.15152, + "63": 0.15303, + "64": 0.1544, + "65": 0.15494, + "66": 0.1538, + "67": 0.15554, + "68": 0.15453, + "69": 0.15381, + "70": 0.15499, + "71": 0.1515, + "72": 0.15265, + "73": 0.15572, + "74": 0.15502, + "75": 0.15343, + "76": 0.15493, + "77": 0.15121, + "78": 0.1558, + "79": 0.15348, + "80": 0.15558, + "81": 0.15602, + "82": 0.15516, + "83": 0.23123, + "84": 0.15567, + "85": 0.15538, + "86": 0.15237, + "87": 0.1529, + "88": 0.15537, + "89": 0.16074, + "90": 0.15689, + "91": 0.15972, + "92": 0.15763, + "93": 0.15401, + "94": 0.15597, + "95": 0.15584, + "96": 0.16537, + "97": 0.15763, + "98": 0.1578, + "99": 0.15295, + "100": 0.15881 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json new file mode 100644 index 00000000000..df866d20398 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80779, + "2": 10.81028, + "3": 10.82881, + "4": 10.82931, + "5": 10.85227, + "6": 10.84112, + "7": 10.83921, + "8": 10.80433, + "9": 10.87802, + "10": 10.88031, + "11": 10.87647, + "12": 10.82798, + "13": 10.84122, + "14": 10.81649, + "15": 10.80572, + "16": 10.7979, + "17": 10.76665, + "18": 10.78004, + "19": 10.7521, + "20": 10.63378, + "21": 10.68086, + "22": 10.63834, + "23": 10.75546, + "24": 10.61158, + "25": 10.47402, + "26": 10.59699, + "27": 10.53929, + "28": 10.45199, + "29": 10.39324, + "30": 10.39328, + "31": 10.49337, + "32": 10.32183, + "33": 10.2767, + "34": 10.44619, + "35": 9.96854, + "36": 10.11277, + "37": 10.02313, + "38": 10.37525, + "39": 9.78827, + "40": 10.10093, + "41": 10.1232, + "42": 10.02925, + "43": 10.19868, + "44": 10.05532, + "45": 9.68289, + "46": 9.98153, + "47": 9.92617, + "48": 9.66965, + "49": 9.91955, + "50": 9.92258 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1058.0, + "2": 1243.0, + "3": 1255.0, + "4": 1229.0, + "5": 1178.0, + "6": 1278.0, + "7": 1129.0, + "8": 955.0, + "9": 1299.0, + "10": 1314.0, + "11": 1220.0, + "12": 1334.0, + "13": 1364.0, + "14": 1153.0, + "15": 1214.0, + "16": 1150.0, + "17": 1176.0, + "18": 1304.0, + "19": 1090.0, + "20": 1170.0, + "21": 1286.0, + "22": 1325.0, + "23": 1353.0, + "24": 1341.0, + "25": 1100.0, + "26": 1174.0, + "27": 1357.0, + "28": 1293.0, + "29": 1259.0, + "30": 1539.0, + "31": 1485.0, + "32": 1422.0, + "33": 1410.0, + "34": 1525.0, + "35": 1163.0, + "36": 1230.0, + "37": 1392.0, + "38": 1570.0, + "39": 1260.0, + "40": 1481.0, + "41": 1687.0, + "42": 1606.0, + "43": 1696.0, + "44": 1648.0, + "45": 1620.0, + "46": 1901.0, + "47": 1627.0, + "48": 1595.0, + "49": 1750.0, + "50": 1723.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 730321408.0, + "2": 730321408.0, + "3": 730321408.0, + "4": 730321408.0, + "5": 730321408.0, + "6": 730321408.0, + "7": 730321408.0, + "8": 730321408.0, + "9": 730321408.0, + "10": 730321408.0, + "11": 730321408.0, + "12": 730321408.0, + "13": 730321408.0, + "14": 730321408.0, + "15": 730321408.0, + "16": 730321408.0, + "17": 730321408.0, + "18": 730321408.0, + "19": 730321408.0, + "20": 730321408.0, + "21": 730321408.0, + "22": 730321408.0, + "23": 730321408.0, + "24": 730321408.0, + "25": 730321408.0, + "26": 730321408.0, + "27": 730321408.0, + "28": 730321408.0, + "29": 730321408.0, + "30": 730321408.0, + "31": 730321408.0, + "32": 730321408.0, + "33": 730321408.0, + "34": 730321408.0, + "35": 730321408.0, + "36": 730321408.0, + "37": 730321408.0, + "38": 730321408.0, + "39": 730321408.0, + "40": 730321408.0, + "41": 730321408.0, + "42": 730321408.0, + "43": 730321408.0, + "44": 730321408.0, + "45": 730321408.0, + "46": 730321408.0, + "47": 730321408.0, + "48": 730321408.0, + "49": 730321408.0, + "50": 730321408.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4311542784.0, + "2": 4593254400.0, + "3": 4593254400.0, + "4": 4593254400.0, + "5": 4593254400.0, + "6": 4593254400.0, + "7": 4593254400.0, + "8": 4593254400.0, + "9": 4593254400.0, + "10": 4593254400.0, + "11": 4593254400.0, + "12": 4593254400.0, + "13": 4593254400.0, + "14": 4593254400.0, + "15": 4593254400.0, + "16": 4593254400.0, + "17": 4593254400.0, + "18": 4593254400.0, + "19": 4593254400.0, + "20": 4593254400.0, + "21": 4593254400.0, + "22": 4593254400.0, + "23": 4593254400.0, + "24": 4593254400.0, + "25": 4593254400.0, + "26": 4593254400.0, + "27": 4593254400.0, + "28": 4593254400.0, + "29": 4593254400.0, + "30": 4593254400.0, + "31": 4593254400.0, + "32": 4593254400.0, + "33": 4593254400.0, + "34": 4593254400.0, + "35": 4593254400.0, + "36": 4593254400.0, + "37": 4593254400.0, + "38": 4593254400.0, + "39": 4593254400.0, + "40": 4593254400.0, + "41": 4593254400.0, + "42": 4593254400.0, + "43": 4593254400.0, + "44": 4593254400.0, + "45": 4593254400.0, + "46": 4593254400.0, + "47": 4593254400.0, + "48": 4593254400.0, + "49": 4593254400.0, + "50": 4593254400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.09337, + "2": 0.2126, + "3": 0.16802, + "4": 0.16952, + "5": 0.16722, + "6": 0.1642, + "7": 0.16788, + "8": 0.16798, + "9": 0.16598, + "10": 0.16531, + "11": 0.16567, + "12": 0.1649, + "13": 0.16886, + "14": 0.16656, + "15": 0.16574, + "16": 0.16557, + "17": 0.16887, + "18": 0.16656, + "19": 0.16927, + "20": 0.16661, + "21": 0.16787, + "22": 0.1648, + "23": 0.16724, + "24": 0.16552, + "25": 0.16584, + "26": 0.16524, + "27": 0.16567, + "28": 0.16594, + "29": 0.16717, + "30": 0.16595, + "31": 0.1685, + "32": 0.1649, + "33": 0.16664, + "34": 0.16525, + "35": 0.16472, + "36": 0.16475, + "37": 0.16492, + "38": 0.16809, + "39": 0.16513, + "40": 0.16748, + "41": 0.16566, + "42": 0.16792, + "43": 0.16514, + "44": 0.16635, + "45": 0.16774, + "46": 0.16907, + "47": 0.16612, + "48": 0.16961, + "49": 0.16797, + "50": 0.16798 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json new file mode 100644 index 00000000000..7b8eff064f8 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.97215, + "2": 10.96419, + "3": 10.96484, + "4": 10.9625, + "5": 10.96069, + "6": 10.96314, + "7": 10.95542, + "8": 10.96012, + "9": 10.9642, + "10": 10.94938, + "11": 10.95571, + "12": 10.94758, + "13": 10.94696, + "14": 10.95407, + "15": 10.91083, + "16": 10.90945, + "17": 10.89001, + "18": 10.89837, + "19": 10.89189, + "20": 10.80856, + "21": 10.78598, + "22": 10.69678, + "23": 10.79518, + "24": 10.6983, + "25": 10.66637, + "26": 10.71173, + "27": 10.68383, + "28": 10.62344, + "29": 10.65067, + "30": 10.45487, + "31": 10.22475, + "32": 10.52171, + "33": 10.52706, + "34": 10.29181, + "35": 10.33574, + "36": 10.2871, + "37": 10.39557, + "38": 10.2536, + "39": 10.44971, + "40": 10.14791, + "41": 10.19255, + "42": 10.24716, + "43": 9.89482, + "44": 10.00764, + "45": 9.88792, + "46": 9.86452, + "47": 10.16578, + "48": 9.87929, + "49": 9.5693, + "50": 9.95204 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727298.0, + "2": 22924540.0, + "3": 22596662.0, + "4": 23219864.0, + "5": 22714604.0, + "6": 23021138.0, + "7": 22770420.0, + "8": 22926748.0, + "9": 22842456.0, + "10": 22918536.0, + "11": 22500950.0, + "12": 22459666.0, + "13": 22917172.0, + "14": 22388760.0, + "15": 22821456.0, + "16": 22831856.0, + "17": 22818966.0, + "18": 22582904.0, + "19": 22618308.0, + "20": 22695018.0, + "21": 22739412.0, + "22": 22799506.0, + "23": 22539964.0, + "24": 22771406.0, + "25": 22818944.0, + "26": 22548888.0, + "27": 22467554.0, + "28": 22453820.0, + "29": 22529468.0, + "30": 22631974.0, + "31": 22954914.0, + "32": 22584176.0, + "33": 22557850.0, + "34": 22835592.0, + "35": 22787628.0, + "36": 22589872.0, + "37": 22497168.0, + "38": 22896192.0, + "39": 22801888.0, + "40": 22658038.0, + "41": 22659420.0, + "42": 22666532.0, + "43": 22976852.0, + "44": 22746072.0, + "45": 22675348.0, + "46": 22884600.0, + "47": 22633528.0, + "48": 22929392.0, + "49": 22728496.0, + "50": 22905024.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 659815936.0, + "2": 659815936.0, + "3": 659815936.0, + "4": 659815936.0, + "5": 659815936.0, + "6": 659815936.0, + "7": 659815936.0, + "8": 659815936.0, + "9": 659815936.0, + "10": 659815936.0, + "11": 659815936.0, + "12": 659815936.0, + "13": 659815936.0, + "14": 659815936.0, + "15": 659815936.0, + "16": 659815936.0, + "17": 659815936.0, + "18": 659815936.0, + "19": 659815936.0, + "20": 659815936.0, + "21": 659815936.0, + "22": 659815936.0, + "23": 659815936.0, + "24": 659815936.0, + "25": 659815936.0, + "26": 659815936.0, + "27": 659815936.0, + "28": 659815936.0, + "29": 659815936.0, + "30": 659815936.0, + "31": 659815936.0, + "32": 659815936.0, + "33": 659815936.0, + "34": 659815936.0, + "35": 659815936.0, + "36": 659815936.0, + "37": 659815936.0, + "38": 659815936.0, + "39": 659815936.0, + "40": 659815936.0, + "41": 659815936.0, + "42": 659815936.0, + "43": 659815936.0, + "44": 659815936.0, + "45": 659815936.0, + "46": 659815936.0, + "47": 659815936.0, + "48": 659815936.0, + "49": 659815936.0, + "50": 659815936.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2128664064.0, + "2": 2387253760.0, + "3": 2387253760.0, + "4": 2387253760.0, + "5": 2387253760.0, + "6": 2387253760.0, + "7": 2387253760.0, + "8": 2387253760.0, + "9": 2387253760.0, + "10": 2387253760.0, + "11": 2387253760.0, + "12": 2387253760.0, + "13": 2387253760.0, + "14": 2387253760.0, + "15": 2387253760.0, + "16": 2387253760.0, + "17": 2387253760.0, + "18": 2387253760.0, + "19": 2387253760.0, + "20": 2387253760.0, + "21": 2387253760.0, + "22": 2387253760.0, + "23": 2387253760.0, + "24": 2387253760.0, + "25": 2387253760.0, + "26": 2387253760.0, + "27": 2387253760.0, + "28": 2387253760.0, + "29": 2387253760.0, + "30": 2387253760.0, + "31": 2387253760.0, + "32": 2387253760.0, + "33": 2387253760.0, + "34": 2387253760.0, + "35": 2387253760.0, + "36": 2387253760.0, + "37": 2387253760.0, + "38": 2387253760.0, + "39": 2387253760.0, + "40": 2387253760.0, + "41": 2387253760.0, + "42": 2387253760.0, + "43": 2387253760.0, + "44": 2387253760.0, + "45": 2387253760.0, + "46": 2387253760.0, + "47": 2387253760.0, + "48": 2387253760.0, + "49": 2387253760.0, + "50": 2387253760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.28374, + "2": 0.19494, + "3": 0.16547, + "4": 0.16142, + "5": 0.16226, + "6": 0.16103, + "7": 0.16344, + "8": 0.1613, + "9": 0.16334, + "10": 0.16281, + "11": 0.1625, + "12": 0.16154, + "13": 0.16348, + "14": 0.16137, + "15": 0.16209, + "16": 0.16137, + "17": 0.16383, + "18": 0.16091, + "19": 0.16265, + "20": 0.16111, + "21": 0.16264, + "22": 0.16338, + "23": 0.16185, + "24": 0.16217, + "25": 0.16378, + "26": 0.16274, + "27": 0.16264, + "28": 0.16343, + "29": 0.16152, + "30": 0.16302, + "31": 0.16591, + "32": 0.1631, + "33": 0.16128, + "34": 0.16402, + "35": 0.16299, + "36": 0.16172, + "37": 0.16136, + "38": 0.1628, + "39": 0.16446, + "40": 0.16262, + "41": 0.16214, + "42": 0.16374, + "43": 0.16424, + "44": 0.16446, + "45": 0.16425, + "46": 0.16386, + "47": 0.16323, + "48": 0.16694, + "49": 0.16551, + "50": 0.16347 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json new file mode 100644 index 00000000000..7978f90dee9 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.84562, + "2": 10.8536, + "3": 10.85978, + "4": 10.84186, + "5": 10.8756, + "6": 10.88172, + "7": 10.86719, + "8": 10.84992, + "9": 10.86792, + "10": 10.83289, + "11": 10.91083, + "12": 10.87501, + "13": 10.86463, + "14": 10.8928, + "15": 10.84145, + "16": 10.84108, + "17": 10.82014, + "18": 10.85327, + "19": 10.8576, + "20": 10.80111, + "21": 10.78705, + "22": 10.72832, + "23": 10.812, + "24": 10.74372, + "25": 10.712, + "26": 10.76908, + "27": 10.78641, + "28": 10.73219, + "29": 10.75717, + "30": 10.58162, + "31": 10.43196, + "32": 10.68369, + "33": 10.66821, + "34": 10.49987, + "35": 10.5319, + "36": 10.52076, + "37": 10.59704, + "38": 10.45745, + "39": 10.62078, + "40": 10.35651, + "41": 10.40224, + "42": 10.45518, + "43": 10.11645, + "44": 10.24176, + "45": 10.1377, + "46": 10.11459, + "47": 10.39867, + "48": 10.14178, + "49": 9.8915, + "50": 10.20004 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 22727180.0, + "2": 22925490.0, + "3": 22595358.0, + "4": 23219094.0, + "5": 22713274.0, + "6": 23020840.0, + "7": 22770150.0, + "8": 22925912.0, + "9": 22842304.0, + "10": 22917686.0, + "11": 22499934.0, + "12": 22458902.0, + "13": 22916544.0, + "14": 22388412.0, + "15": 22820828.0, + "16": 22829442.0, + "17": 22818424.0, + "18": 22582572.0, + "19": 22616516.0, + "20": 22693786.0, + "21": 22738788.0, + "22": 22800098.0, + "23": 22538252.0, + "24": 22771304.0, + "25": 22818714.0, + "26": 22547732.0, + "27": 22467548.0, + "28": 22452916.0, + "29": 22528584.0, + "30": 22630192.0, + "31": 22954950.0, + "32": 22584400.0, + "33": 22557234.0, + "34": 22834292.0, + "35": 22786428.0, + "36": 22588576.0, + "37": 22496998.0, + "38": 22895112.0, + "39": 22800900.0, + "40": 22657380.0, + "41": 22658838.0, + "42": 22666328.0, + "43": 22975596.0, + "44": 22745924.0, + "45": 22674268.0, + "46": 22884128.0, + "47": 22632352.0, + "48": 22927496.0, + "49": 22727204.0, + "50": 22903716.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 609364480.0, + "2": 609364480.0, + "3": 609364480.0, + "4": 609364480.0, + "5": 609364480.0, + "6": 609364480.0, + "7": 609364480.0, + "8": 609364480.0, + "9": 609364480.0, + "10": 609364480.0, + "11": 609364480.0, + "12": 609364480.0, + "13": 609364480.0, + "14": 609364480.0, + "15": 609364480.0, + "16": 609364480.0, + "17": 609364480.0, + "18": 609364480.0, + "19": 609364480.0, + "20": 609364480.0, + "21": 609364480.0, + "22": 609364480.0, + "23": 609364480.0, + "24": 609364480.0, + "25": 609364480.0, + "26": 609364480.0, + "27": 609364480.0, + "28": 609364480.0, + "29": 609364480.0, + "30": 609364480.0, + "31": 609364480.0, + "32": 609364480.0, + "33": 609364480.0, + "34": 609364480.0, + "35": 609364480.0, + "36": 609364480.0, + "37": 609364480.0, + "38": 609364480.0, + "39": 609364480.0, + "40": 609364480.0, + "41": 609364480.0, + "42": 609364480.0, + "43": 609364480.0, + "44": 609364480.0, + "45": 609364480.0, + "46": 609364480.0, + "47": 609364480.0, + "48": 609364480.0, + "49": 609364480.0, + "50": 609364480.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2580665856.0, + "2": 2810890752.0, + "3": 2811808256.0, + "4": 2811808256.0, + "5": 2811808256.0, + "6": 2811808256.0, + "7": 2811808256.0, + "8": 2811808256.0, + "9": 2811808256.0, + "10": 2811808256.0, + "11": 2811808256.0, + "12": 2811808256.0, + "13": 2811808256.0, + "14": 2811808256.0, + "15": 2811808256.0, + "16": 2811808256.0, + "17": 2811808256.0, + "18": 2811808256.0, + "19": 2811808256.0, + "20": 2811808256.0, + "21": 2811808256.0, + "22": 2811808256.0, + "23": 2811808256.0, + "24": 2811808256.0, + "25": 2811808256.0, + "26": 2811808256.0, + "27": 2811808256.0, + "28": 2811808256.0, + "29": 2811808256.0, + "30": 2811808256.0, + "31": 2811808256.0, + "32": 2811808256.0, + "33": 2811808256.0, + "34": 2811808256.0, + "35": 2811808256.0, + "36": 2811808256.0, + "37": 2811808256.0, + "38": 2811808256.0, + "39": 2811808256.0, + "40": 2811808256.0, + "41": 2811808256.0, + "42": 2811808256.0, + "43": 2811808256.0, + "44": 2811808256.0, + "45": 2811808256.0, + "46": 2811808256.0, + "47": 2811808256.0, + "48": 2811808256.0, + "49": 2811808256.0, + "50": 2811808256.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 8.75982, + "2": 0.12447, + "3": 0.09544, + "4": 0.09457, + "5": 0.09468, + "6": 0.09408, + "7": 0.09258, + "8": 0.0918, + "9": 0.09183, + "10": 0.09166, + "11": 0.09114, + "12": 0.09162, + "13": 0.09203, + "14": 0.09276, + "15": 0.09287, + "16": 0.09143, + "17": 0.09172, + "18": 0.09105, + "19": 0.0913, + "20": 0.09127, + "21": 0.09171, + "22": 0.09198, + "23": 0.09174, + "24": 0.09202, + "25": 0.09194, + "26": 0.09204, + "27": 0.0918, + "28": 0.09189, + "29": 0.09173, + "30": 0.09217, + "31": 0.09241, + "32": 0.09169, + "33": 0.09153, + "34": 0.09222, + "35": 0.09221, + "36": 0.09178, + "37": 0.0919, + "38": 0.09176, + "39": 0.09172, + "40": 0.09191, + "41": 0.09185, + "42": 0.09204, + "43": 0.09234, + "44": 0.09243, + "45": 0.09236, + "46": 0.09235, + "47": 0.09258, + "48": 0.09291, + "49": 0.09204, + "50": 0.09226 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json new file mode 100644 index 00000000000..f3b14941f76 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88765, + "2": 10.90324, + "3": 10.87803, + "4": 10.87967, + "5": 10.91013, + "6": 10.91377, + "7": 10.89285, + "8": 10.88408, + "9": 10.89749, + "10": 10.87762, + "11": 10.88984, + "12": 10.89502, + "13": 10.90738, + "14": 10.90831, + "15": 10.86285, + "16": 10.85944, + "17": 10.8221, + "18": 10.84343, + "19": 10.83663, + "20": 10.74694, + "21": 10.73044, + "22": 10.62623, + "23": 10.74379, + "24": 10.63229, + "25": 10.59789, + "26": 10.64585, + "27": 10.64828, + "28": 10.59539, + "29": 10.60615, + "30": 10.40161, + "31": 10.18116, + "32": 10.49112, + "33": 10.48508, + "34": 10.25199, + "35": 10.30784, + "36": 10.25604, + "37": 10.36576, + "38": 10.21673, + "39": 10.40989, + "40": 10.11047, + "41": 10.15883, + "42": 10.22753, + "43": 9.8745, + "44": 9.97803, + "45": 9.8591, + "46": 9.8546, + "47": 10.15264, + "48": 9.86489, + "49": 9.5555, + "50": 9.92088, + "51": 9.86094, + "52": 9.75697, + "53": 10.07633, + "54": 9.96082, + "55": 9.88565, + "56": 9.6349, + "57": 9.4925, + "58": 9.83099, + "59": 9.59122, + "60": 9.50798, + "61": 9.7061, + "62": 9.98413, + "63": 9.37604, + "64": 9.77938, + "65": 8.95852, + "66": 9.70596, + "67": 9.37402, + "68": 9.78683, + "69": 9.78932, + "70": 9.72766, + "71": 9.61135, + "72": 9.59178, + "73": 9.49896, + "74": 8.95742, + "75": 9.42469, + "76": 9.09651, + "77": 10.06653, + "78": 9.73149, + "79": 9.37959, + "80": 9.40394, + "81": 9.48277, + "82": 9.69318, + "83": 9.31104, + "84": 9.4139, + "85": 9.61469, + "86": 9.07793, + "87": 9.59662, + "88": 9.74827, + "89": 9.60196, + "90": 9.81239, + "91": 9.34524, + "92": 9.36524, + "93": 9.07745, + "94": 8.83182, + "95": 9.521, + "96": 9.52525, + "97": 9.31322, + "98": 9.677, + "99": 8.88904, + "100": 9.40063 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1753.0, + "2": 1863.0, + "3": 1781.0, + "4": 1850.0, + "5": 2021.0, + "6": 1987.0, + "7": 2006.0, + "8": 1657.0, + "9": 1961.0, + "10": 1491.0, + "11": 2010.0, + "12": 1846.0, + "13": 1934.0, + "14": 1954.0, + "15": 1942.0, + "16": 1969.0, + "17": 1936.0, + "18": 1836.0, + "19": 1841.0, + "20": 1684.0, + "21": 1958.0, + "22": 1782.0, + "23": 2112.0, + "24": 1651.0, + "25": 1713.0, + "26": 1878.0, + "27": 1837.0, + "28": 2025.0, + "29": 2115.0, + "30": 1867.0, + "31": 1622.0, + "32": 1939.0, + "33": 2167.0, + "34": 1867.0, + "35": 2035.0, + "36": 2086.0, + "37": 2477.0, + "38": 2296.0, + "39": 2505.0, + "40": 2150.0, + "41": 2315.0, + "42": 2239.0, + "43": 1941.0, + "44": 2169.0, + "45": 2172.0, + "46": 2231.0, + "47": 2604.0, + "48": 2429.0, + "49": 2289.0, + "50": 2529.0, + "51": 2742.0, + "52": 2671.0, + "53": 3066.0, + "54": 2782.0, + "55": 2510.0, + "56": 2874.0, + "57": 2304.0, + "58": 3111.0, + "59": 2862.0, + "60": 2374.0, + "61": 2977.0, + "62": 2740.0, + "63": 2394.0, + "64": 3232.0, + "65": 2720.0, + "66": 3277.0, + "67": 2810.0, + "68": 2830.0, + "69": 3094.0, + "70": 3327.0, + "71": 3106.0, + "72": 2261.0, + "73": 3147.0, + "74": 1902.0, + "75": 2545.0, + "76": 2905.0, + "77": 3468.0, + "78": 3432.0, + "79": 3336.0, + "80": 3434.0, + "81": 3605.0, + "82": 3269.0, + "83": 2891.0, + "84": 3343.0, + "85": 3501.0, + "86": 2786.0, + "87": 3872.0, + "88": 3019.0, + "89": 3407.0, + "90": 3023.0, + "91": 2630.0, + "92": 3186.0, + "93": 2746.0, + "94": 3526.0, + "95": 3414.0, + "96": 3546.0, + "97": 3339.0, + "98": 3758.0, + "99": 3058.0, + "100": 3454.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 462388736.0, + "2": 462388736.0, + "3": 462388736.0, + "4": 462388736.0, + "5": 462388736.0, + "6": 462388736.0, + "7": 462388736.0, + "8": 462388736.0, + "9": 462388736.0, + "10": 462388736.0, + "11": 462388736.0, + "12": 462388736.0, + "13": 462388736.0, + "14": 462388736.0, + "15": 462388736.0, + "16": 462388736.0, + "17": 462388736.0, + "18": 462388736.0, + "19": 462388736.0, + "20": 462388736.0, + "21": 462388736.0, + "22": 462388736.0, + "23": 462388736.0, + "24": 462388736.0, + "25": 462388736.0, + "26": 462388736.0, + "27": 462388736.0, + "28": 462388736.0, + "29": 462388736.0, + "30": 462388736.0, + "31": 462388736.0, + "32": 462388736.0, + "33": 462388736.0, + "34": 462388736.0, + "35": 462388736.0, + "36": 462388736.0, + "37": 462388736.0, + "38": 462388736.0, + "39": 462388736.0, + "40": 462388736.0, + "41": 462388736.0, + "42": 462388736.0, + "43": 462388736.0, + "44": 462388736.0, + "45": 462388736.0, + "46": 462388736.0, + "47": 462388736.0, + "48": 462388736.0, + "49": 462388736.0, + "50": 462388736.0, + "51": 462388736.0, + "52": 462388736.0, + "53": 462388736.0, + "54": 462388736.0, + "55": 462388736.0, + "56": 462388736.0, + "57": 462388736.0, + "58": 462388736.0, + "59": 462388736.0, + "60": 462388736.0, + "61": 462388736.0, + "62": 462388736.0, + "63": 462388736.0, + "64": 462388736.0, + "65": 462388736.0, + "66": 462388736.0, + "67": 462388736.0, + "68": 462388736.0, + "69": 462388736.0, + "70": 462388736.0, + "71": 462388736.0, + "72": 462388736.0, + "73": 462388736.0, + "74": 462388736.0, + "75": 462388736.0, + "76": 462388736.0, + "77": 462388736.0, + "78": 462388736.0, + "79": 462388736.0, + "80": 462388736.0, + "81": 462388736.0, + "82": 462388736.0, + "83": 462388736.0, + "84": 462388736.0, + "85": 462388736.0, + "86": 462388736.0, + "87": 462388736.0, + "88": 462388736.0, + "89": 462388736.0, + "90": 462388736.0, + "91": 462388736.0, + "92": 462388736.0, + "93": 462388736.0, + "94": 462388736.0, + "95": 462388736.0, + "96": 462388736.0, + "97": 462388736.0, + "98": 462388736.0, + "99": 462388736.0, + "100": 462388736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1549081088.0, + "2": 1677508096.0, + "3": 1677508096.0, + "4": 1677508096.0, + "5": 1677508096.0, + "6": 1677508096.0, + "7": 1677508096.0, + "8": 1677508096.0, + "9": 1677508096.0, + "10": 1677508096.0, + "11": 1677508096.0, + "12": 1677508096.0, + "13": 1677508096.0, + "14": 1677508096.0, + "15": 1677508096.0, + "16": 1677508096.0, + "17": 1677508096.0, + "18": 1677508096.0, + "19": 1677508096.0, + "20": 1677508096.0, + "21": 1677508096.0, + "22": 1677508096.0, + "23": 1677508096.0, + "24": 1677508096.0, + "25": 1677508096.0, + "26": 1677508096.0, + "27": 1677508096.0, + "28": 1677508096.0, + "29": 1677508096.0, + "30": 1677508096.0, + "31": 1677508096.0, + "32": 1677508096.0, + "33": 1677508096.0, + "34": 1677508096.0, + "35": 1677508096.0, + "36": 1677508096.0, + "37": 1677508096.0, + "38": 1677508096.0, + "39": 1677508096.0, + "40": 1677508096.0, + "41": 1677508096.0, + "42": 1677508096.0, + "43": 1677508096.0, + "44": 1677508096.0, + "45": 1677508096.0, + "46": 1677508096.0, + "47": 1677508096.0, + "48": 1677508096.0, + "49": 1677508096.0, + "50": 1677508096.0, + "51": 1677508096.0, + "52": 1677508096.0, + "53": 1677508096.0, + "54": 1677508096.0, + "55": 1677508096.0, + "56": 1677508096.0, + "57": 1677508096.0, + "58": 1677508096.0, + "59": 1677508096.0, + "60": 1677508096.0, + "61": 1677508096.0, + "62": 1677508096.0, + "63": 1677508096.0, + "64": 1677508096.0, + "65": 1677508096.0, + "66": 1677508096.0, + "67": 1677508096.0, + "68": 1677508096.0, + "69": 1677508096.0, + "70": 1677508096.0, + "71": 1677508096.0, + "72": 1677508096.0, + "73": 1677508096.0, + "74": 1677508096.0, + "75": 1677508096.0, + "76": 1677508096.0, + "77": 1677508096.0, + "78": 1677508096.0, + "79": 1677508096.0, + "80": 1677508096.0, + "81": 1677508096.0, + "82": 1677508096.0, + "83": 1677508096.0, + "84": 1677508096.0, + "85": 1677508096.0, + "86": 1677508096.0, + "87": 1677508096.0, + "88": 1677508096.0, + "89": 1677508096.0, + "90": 1677508096.0, + "91": 1677508096.0, + "92": 1677508096.0, + "93": 1677508096.0, + "94": 1677508096.0, + "95": 1677508096.0, + "96": 1677508096.0, + "97": 1677508096.0, + "98": 1677508096.0, + "99": 1677508096.0, + "100": 1677508096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.99944, + "2": 0.20045, + "3": 0.19793, + "4": 0.17848, + "5": 0.16964, + "6": 0.16468, + "7": 0.16476, + "8": 0.16483, + "9": 0.16623, + "10": 0.16654, + "11": 0.16538, + "12": 0.16553, + "13": 0.16914, + "14": 0.16787, + "15": 0.16661, + "16": 0.16662, + "17": 0.16726, + "18": 0.16649, + "19": 0.16762, + "20": 0.16602, + "21": 0.16551, + "22": 0.16634, + "23": 0.16637, + "24": 0.16562, + "25": 0.16606, + "26": 0.16615, + "27": 0.16594, + "28": 0.16609, + "29": 0.16618, + "30": 0.1661, + "31": 0.16608, + "32": 0.16596, + "33": 0.16569, + "34": 0.16643, + "35": 0.16649, + "36": 0.16641, + "37": 0.16638, + "38": 0.16654, + "39": 0.16579, + "40": 0.16554, + "41": 0.16559, + "42": 0.16619, + "43": 0.16594, + "44": 0.16595, + "45": 0.16565, + "46": 0.16567, + "47": 0.167, + "48": 0.16607, + "49": 0.16599, + "50": 0.16599, + "51": 0.17281, + "52": 0.1691, + "53": 0.16798, + "54": 0.16863, + "55": 0.16788, + "56": 0.16665, + "57": 0.16953, + "58": 0.16775, + "59": 0.16846, + "60": 0.16838, + "61": 0.16768, + "62": 0.16906, + "63": 0.1669, + "64": 0.16771, + "65": 0.16771, + "66": 0.16746, + "67": 0.16745, + "68": 0.16835, + "69": 0.1684, + "70": 0.16828, + "71": 0.16771, + "72": 0.16752, + "73": 0.16809, + "74": 0.1675, + "75": 0.16907, + "76": 0.16756, + "77": 0.16748, + "78": 0.16798, + "79": 0.16778, + "80": 0.1673, + "81": 0.16801, + "82": 0.16821, + "83": 0.16768, + "84": 0.16782, + "85": 0.16823, + "86": 0.16832, + "87": 0.16791, + "88": 0.16725, + "89": 0.16748, + "90": 0.16753, + "91": 0.16813, + "92": 0.17012, + "93": 0.16753, + "94": 0.16884, + "95": 0.16856, + "96": 0.16826, + "97": 0.16856, + "98": 0.1684, + "99": 0.16853, + "100": 0.16844 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..47def03f85b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.93732, + "2": 0.23227, + "3": 0.20089, + "4": 0.20035, + "5": 0.19896, + "6": 0.19915, + "7": 0.19862, + "8": 0.19876, + "9": 0.19839, + "10": 0.19916, + "11": 0.19842, + "12": 0.19799, + "13": 0.19859, + "14": 0.19899, + "15": 0.20375, + "16": 0.20118, + "17": 0.20189, + "18": 0.19857, + "19": 0.19946, + "20": 0.1998, + "21": 0.19879, + "22": 0.2041, + "23": 0.20368, + "24": 0.19956, + "25": 0.19992, + "26": 0.19925, + "27": 0.19925, + "28": 0.19792, + "29": 0.20035, + "30": 0.19833, + "31": 0.1958, + "32": 0.22529, + "33": 0.19995, + "34": 0.19815, + "35": 0.1985, + "36": 0.19811, + "37": 0.1987, + "38": 0.19926, + "39": 0.1984, + "40": 0.1994, + "41": 0.19971, + "42": 0.19908, + "43": 0.19879, + "44": 0.19844, + "45": 0.19859, + "46": 0.20014, + "47": 0.19821, + "48": 0.19799, + "49": 0.19877, + "50": 0.19868, + "51": 0.24766, + "52": 0.21197, + "53": 0.20757, + "54": 0.2054, + "55": 0.20708, + "56": 0.20792, + "57": 0.20801, + "58": 0.21159, + "59": 0.19937, + "60": 0.19687, + "61": 0.19676, + "62": 0.19746, + "63": 0.21775, + "64": 0.19712, + "65": 0.19664, + "66": 0.19704, + "67": 0.19709, + "68": 0.197, + "69": 0.19493, + "70": 0.19547, + "71": 0.19588, + "72": 0.19631, + "73": 0.19693, + "74": 0.19604, + "75": 0.19587, + "76": 0.19475, + "77": 0.1958, + "78": 0.19556, + "79": 0.19562, + "80": 0.19508, + "81": 0.19396, + "82": 0.19391, + "83": 0.19488, + "84": 0.19509, + "85": 0.21114, + "86": 0.19572, + "87": 0.19485, + "88": 0.19489, + "89": 0.19484, + "90": 0.19431, + "91": 0.19448, + "92": 0.19436, + "93": 0.1942, + "94": 0.19402, + "95": 0.19397, + "96": 0.19454, + "97": 0.19408, + "98": 0.19502, + "99": 0.19427, + "100": 0.19378 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..e882d67fe97 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 487096832.0, + "2": 487096832.0, + "3": 487096832.0, + "4": 487096832.0, + "5": 487096832.0, + "6": 487096832.0, + "7": 487096832.0, + "8": 487096832.0, + "9": 487096832.0, + "10": 487096832.0, + "11": 487096832.0, + "12": 487096832.0, + "13": 487096832.0, + "14": 487096832.0, + "15": 487096832.0, + "16": 487096832.0, + "17": 487096832.0, + "18": 487096832.0, + "19": 487096832.0, + "20": 487096832.0, + "21": 487096832.0, + "22": 487096832.0, + "23": 487096832.0, + "24": 487096832.0, + "25": 487096832.0, + "26": 487096832.0, + "27": 487096832.0, + "28": 487096832.0, + "29": 487096832.0, + "30": 487096832.0, + "31": 487096832.0, + "32": 487096832.0, + "33": 487096832.0, + "34": 487096832.0, + "35": 487096832.0, + "36": 487096832.0, + "37": 487096832.0, + "38": 487096832.0, + "39": 487096832.0, + "40": 487096832.0, + "41": 487096832.0, + "42": 487096832.0, + "43": 487096832.0, + "44": 487096832.0, + "45": 487096832.0, + "46": 487096832.0, + "47": 487096832.0, + "48": 487096832.0, + "49": 487096832.0, + "50": 487096832.0, + "51": 487096832.0, + "52": 487096832.0, + "53": 487096832.0, + "54": 487096832.0, + "55": 487096832.0, + "56": 487096832.0, + "57": 487096832.0, + "58": 487096832.0, + "59": 487096832.0, + "60": 487096832.0, + "61": 487096832.0, + "62": 487096832.0, + "63": 487096832.0, + "64": 487096832.0, + "65": 487096832.0, + "66": 487096832.0, + "67": 487096832.0, + "68": 487096832.0, + "69": 487096832.0, + "70": 487096832.0, + "71": 487096832.0, + "72": 487096832.0, + "73": 487096832.0, + "74": 487096832.0, + "75": 487096832.0, + "76": 487096832.0, + "77": 487096832.0, + "78": 487096832.0, + "79": 487096832.0, + "80": 487096832.0, + "81": 487096832.0, + "82": 487096832.0, + "83": 487096832.0, + "84": 487096832.0, + "85": 487096832.0, + "86": 487096832.0, + "87": 487096832.0, + "88": 487096832.0, + "89": 487096832.0, + "90": 487096832.0, + "91": 487096832.0, + "92": 487096832.0, + "93": 487096832.0, + "94": 487096832.0, + "95": 487096832.0, + "96": 487096832.0, + "97": 487096832.0, + "98": 487096832.0, + "99": 487096832.0, + "100": 487096832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1228699136.0, + "2": 1409821184.0, + "3": 1409821184.0, + "4": 1409821184.0, + "5": 1409821184.0, + "6": 1409821184.0, + "7": 1409821184.0, + "8": 1409821184.0, + "9": 1409821184.0, + "10": 1409821184.0, + "11": 1409821184.0, + "12": 1409821184.0, + "13": 1409821184.0, + "14": 1409821184.0, + "15": 1409821184.0, + "16": 1409821184.0, + "17": 1409821184.0, + "18": 1409821184.0, + "19": 1409821184.0, + "20": 1409821184.0, + "21": 1409821184.0, + "22": 1409821184.0, + "23": 1409821184.0, + "24": 1409821184.0, + "25": 1409821184.0, + "26": 1409821184.0, + "27": 1409821184.0, + "28": 1409821184.0, + "29": 1409821184.0, + "30": 1409821184.0, + "31": 1409821184.0, + "32": 1409821184.0, + "33": 1409821184.0, + "34": 1409821184.0, + "35": 1409821184.0, + "36": 1409821184.0, + "37": 1409821184.0, + "38": 1409821184.0, + "39": 1409821184.0, + "40": 1409821184.0, + "41": 1409821184.0, + "42": 1409821184.0, + "43": 1409821184.0, + "44": 1409821184.0, + "45": 1409821184.0, + "46": 1409821184.0, + "47": 1409821184.0, + "48": 1409821184.0, + "49": 1409821184.0, + "50": 1409821184.0, + "51": 1409821184.0, + "52": 1409821184.0, + "53": 1409821184.0, + "54": 1409821184.0, + "55": 1409821184.0, + "56": 1409821184.0, + "57": 1409821184.0, + "58": 1409821184.0, + "59": 1409821184.0, + "60": 1409821184.0, + "61": 1409821184.0, + "62": 1409821184.0, + "63": 1409821184.0, + "64": 1409821184.0, + "65": 1409821184.0, + "66": 1409821184.0, + "67": 1409821184.0, + "68": 1409821184.0, + "69": 1409821184.0, + "70": 1409821184.0, + "71": 1409821184.0, + "72": 1409821184.0, + "73": 1409821184.0, + "74": 1409821184.0, + "75": 1409821184.0, + "76": 1409821184.0, + "77": 1409821184.0, + "78": 1409821184.0, + "79": 1409821184.0, + "80": 1409821184.0, + "81": 1409821184.0, + "82": 1409821184.0, + "83": 1409821184.0, + "84": 1409821184.0, + "85": 1409821184.0, + "86": 1409821184.0, + "87": 1409821184.0, + "88": 1409821184.0, + "89": 1409821184.0, + "90": 1409821184.0, + "91": 1409821184.0, + "92": 1409821184.0, + "93": 1409821184.0, + "94": 1409821184.0, + "95": 1409821184.0, + "96": 1409821184.0, + "97": 1409821184.0, + "98": 1409821184.0, + "99": 1409821184.0, + "100": 1409821184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 8.77229, + "2": 0.2448, + "3": 0.20253, + "4": 0.20053, + "5": 0.20051, + "6": 0.20015, + "7": 0.20192, + "8": 0.20169, + "9": 0.20018, + "10": 0.20165, + "11": 0.20105, + "12": 0.20063, + "13": 0.20054, + "14": 0.20134, + "15": 0.201, + "16": 0.20077, + "17": 0.20014, + "18": 0.20077, + "19": 0.20119, + "20": 0.20025, + "21": 0.1998, + "22": 0.19982, + "23": 0.20717, + "24": 0.20458, + "25": 0.20416, + "26": 0.20401, + "27": 0.20359, + "28": 0.20411, + "29": 0.20461, + "30": 0.2031, + "31": 0.20841, + "32": 0.20513, + "33": 0.20423, + "34": 0.20522, + "35": 0.20462, + "36": 0.20527, + "37": 0.20698, + "38": 0.20431, + "39": 0.20455, + "40": 0.20559, + "41": 0.20473, + "42": 0.20377, + "43": 0.20415, + "44": 0.20517, + "45": 0.20553, + "46": 0.20469, + "47": 0.2048, + "48": 0.20614, + "49": 0.20503, + "50": 0.20599, + "51": 0.23019, + "52": 0.22379, + "53": 0.22131, + "54": 0.22162, + "55": 0.22183, + "56": 0.22134, + "57": 0.22168, + "58": 0.22122, + "59": 0.22115, + "60": 0.22101, + "61": 0.22083, + "62": 0.22107, + "63": 0.22167, + "64": 0.22088, + "65": 0.22041, + "66": 0.22189, + "67": 0.22139, + "68": 0.22099, + "69": 0.22131, + "70": 0.2213, + "71": 0.22124, + "72": 0.22157, + "73": 0.22217, + "74": 0.22095, + "75": 0.22138, + "76": 0.22108, + "77": 0.22085, + "78": 0.2207, + "79": 0.22071, + "80": 0.22059, + "81": 0.22136, + "82": 0.22155, + "83": 0.22071, + "84": 0.22146, + "85": 0.22157, + "86": 0.2209, + "87": 0.22086, + "88": 0.22103, + "89": 0.2209, + "90": 0.22117, + "91": 0.22191, + "92": 0.22124, + "93": 0.22107, + "94": 0.22101, + "95": 0.22098, + "96": 0.22098, + "97": 0.22164, + "98": 0.22195, + "99": 0.22156, + "100": 0.22188 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json new file mode 100644 index 00000000000..99c819db256 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90497, + "5": 10.92805, + "6": 10.93671, + "7": 10.90403, + "8": 10.92231, + "9": 10.91252, + "10": 10.90851, + "11": 10.89338, + "12": 10.9208, + "13": 10.91494, + "14": 10.92144, + "15": 10.88435, + "16": 10.87458, + "17": 10.83918, + "18": 10.87307, + "19": 10.85328, + "20": 10.77494, + "21": 10.74759, + "22": 10.63148, + "23": 10.75623, + "24": 10.65568, + "25": 10.59221, + "26": 10.65325, + "27": 10.64881, + "28": 10.59654, + "29": 10.61012, + "30": 10.39283, + "31": 10.15719, + "32": 10.49221, + "33": 10.47943, + "34": 10.24015, + "35": 10.29711, + "36": 10.24562, + "37": 10.35281, + "38": 10.20533, + "39": 10.40423, + "40": 10.09553, + "41": 10.15279, + "42": 10.21879, + "43": 9.85526, + "44": 9.96244, + "45": 9.84617, + "46": 9.83798, + "47": 10.1388, + "48": 9.85687, + "49": 9.53746, + "50": 9.90877, + "51": 9.84971, + "52": 9.74156, + "53": 10.06324, + "54": 9.94584, + "55": 9.87735, + "56": 9.62744, + "57": 9.4726, + "58": 9.82907, + "59": 9.58298, + "60": 9.49182, + "61": 9.6996, + "62": 9.98091, + "63": 9.37212, + "64": 9.77558, + "65": 8.94327, + "66": 9.69991, + "67": 9.3641, + "68": 9.78706, + "69": 9.78396, + "70": 9.72291, + "71": 9.60749, + "72": 9.58417, + "73": 9.4909, + "74": 8.94863, + "75": 9.41807, + "76": 9.08721, + "77": 10.06284, + "78": 9.729, + "79": 9.37087, + "80": 9.40029, + "81": 9.47753, + "82": 9.69123, + "83": 9.30764, + "84": 9.4125, + "85": 9.61132, + "86": 9.07624, + "87": 9.59459, + "88": 9.74769, + "89": 9.60678, + "90": 9.81079, + "91": 9.34443, + "92": 9.36534, + "93": 9.07741, + "94": 8.82974, + "95": 9.51676, + "96": 9.52545, + "97": 9.31031, + "98": 9.67811, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 59.0, + "2": 59.0, + "3": 57.0, + "4": 55.0, + "5": 68.0, + "6": 60.0, + "7": 58.0, + "8": 63.0, + "9": 62.0, + "10": 47.0, + "11": 57.0, + "12": 48.0, + "13": 68.0, + "14": 58.0, + "15": 75.0, + "16": 67.0, + "17": 59.0, + "18": 56.0, + "19": 72.0, + "20": 52.0, + "21": 60.0, + "22": 69.0, + "23": 77.0, + "24": 58.0, + "25": 71.0, + "26": 68.0, + "27": 55.0, + "28": 94.0, + "29": 57.0, + "30": 66.0, + "31": 60.0, + "32": 64.0, + "33": 69.0, + "34": 73.0, + "35": 73.0, + "36": 72.0, + "37": 89.0, + "38": 72.0, + "39": 75.0, + "40": 82.0, + "41": 82.0, + "42": 74.0, + "43": 90.0, + "44": 69.0, + "45": 83.0, + "46": 68.0, + "47": 74.0, + "48": 95.0, + "49": 81.0, + "50": 83.0, + "51": 77.0, + "52": 100.0, + "53": 71.0, + "54": 67.0, + "55": 70.0, + "56": 83.0, + "57": 74.0, + "58": 106.0, + "59": 72.0, + "60": 98.0, + "61": 67.0, + "62": 73.0, + "63": 77.0, + "64": 94.0, + "65": 82.0, + "66": 87.0, + "67": 65.0, + "68": 78.0, + "69": 59.0, + "70": 102.0, + "71": 82.0, + "72": 60.0, + "73": 96.0, + "74": 61.0, + "75": 64.0, + "76": 70.0, + "77": 84.0, + "78": 93.0, + "79": 102.0, + "80": 71.0, + "81": 88.0, + "82": 85.0, + "83": 75.0, + "84": 69.0, + "85": 84.0, + "86": 66.0, + "87": 93.0, + "88": 96.0, + "89": 73.0, + "90": 77.0, + "91": 66.0, + "92": 86.0, + "93": 63.0, + "94": 60.0, + "95": 70.0, + "96": 65.0, + "97": 67.0, + "98": 96.0, + "99": 54.0, + "100": 77.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 516063744.0, + "2": 516063744.0, + "3": 516063744.0, + "4": 516063744.0, + "5": 516063744.0, + "6": 516063744.0, + "7": 516063744.0, + "8": 516063744.0, + "9": 516063744.0, + "10": 516063744.0, + "11": 516063744.0, + "12": 516063744.0, + "13": 516063744.0, + "14": 516063744.0, + "15": 516063744.0, + "16": 516063744.0, + "17": 516063744.0, + "18": 516063744.0, + "19": 516063744.0, + "20": 516063744.0, + "21": 516063744.0, + "22": 516063744.0, + "23": 516063744.0, + "24": 516063744.0, + "25": 516063744.0, + "26": 516063744.0, + "27": 516063744.0, + "28": 516063744.0, + "29": 516063744.0, + "30": 516063744.0, + "31": 516063744.0, + "32": 516063744.0, + "33": 516063744.0, + "34": 516063744.0, + "35": 516063744.0, + "36": 516063744.0, + "37": 516063744.0, + "38": 516063744.0, + "39": 516063744.0, + "40": 516063744.0, + "41": 516063744.0, + "42": 516063744.0, + "43": 516063744.0, + "44": 516063744.0, + "45": 516063744.0, + "46": 516063744.0, + "47": 516063744.0, + "48": 516063744.0, + "49": 516063744.0, + "50": 516063744.0, + "51": 516063744.0, + "52": 516063744.0, + "53": 516063744.0, + "54": 516063744.0, + "55": 516063744.0, + "56": 516063744.0, + "57": 516063744.0, + "58": 516063744.0, + "59": 516063744.0, + "60": 516063744.0, + "61": 516063744.0, + "62": 516063744.0, + "63": 516063744.0, + "64": 516063744.0, + "65": 516063744.0, + "66": 516063744.0, + "67": 516063744.0, + "68": 516063744.0, + "69": 516063744.0, + "70": 516063744.0, + "71": 516063744.0, + "72": 516063744.0, + "73": 516063744.0, + "74": 516063744.0, + "75": 516063744.0, + "76": 516063744.0, + "77": 516063744.0, + "78": 516063744.0, + "79": 516063744.0, + "80": 516063744.0, + "81": 516063744.0, + "82": 516063744.0, + "83": 516063744.0, + "84": 516063744.0, + "85": 516063744.0, + "86": 516063744.0, + "87": 516063744.0, + "88": 516063744.0, + "89": 516063744.0, + "90": 516063744.0, + "91": 516063744.0, + "92": 516063744.0, + "93": 516063744.0, + "94": 516063744.0, + "95": 516063744.0, + "96": 516063744.0, + "97": 516063744.0, + "98": 516063744.0, + "99": 516063744.0, + "100": 516063744.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1697019392.0, + "2": 1877092864.0, + "3": 1877092864.0, + "4": 1877092864.0, + "5": 1877092864.0, + "6": 1877092864.0, + "7": 1877092864.0, + "8": 1877092864.0, + "9": 1877092864.0, + "10": 1877092864.0, + "11": 1877092864.0, + "12": 1877092864.0, + "13": 1877092864.0, + "14": 1877092864.0, + "15": 1877092864.0, + "16": 1877092864.0, + "17": 1877092864.0, + "18": 1877092864.0, + "19": 1877092864.0, + "20": 1877092864.0, + "21": 1877092864.0, + "22": 1877092864.0, + "23": 1877092864.0, + "24": 1877092864.0, + "25": 1877092864.0, + "26": 1877092864.0, + "27": 1877092864.0, + "28": 1877092864.0, + "29": 1877092864.0, + "30": 1877092864.0, + "31": 1877092864.0, + "32": 1877092864.0, + "33": 1877092864.0, + "34": 1877092864.0, + "35": 1877092864.0, + "36": 1877092864.0, + "37": 1877092864.0, + "38": 1877092864.0, + "39": 1877092864.0, + "40": 1877092864.0, + "41": 1877092864.0, + "42": 1877092864.0, + "43": 1877092864.0, + "44": 1877092864.0, + "45": 1877092864.0, + "46": 1877092864.0, + "47": 1877092864.0, + "48": 1877092864.0, + "49": 1877092864.0, + "50": 1877092864.0, + "51": 1877092864.0, + "52": 1877092864.0, + "53": 1877092864.0, + "54": 1877092864.0, + "55": 1877092864.0, + "56": 1877092864.0, + "57": 1877092864.0, + "58": 1877092864.0, + "59": 1877092864.0, + "60": 1877092864.0, + "61": 1877092864.0, + "62": 1877092864.0, + "63": 1877092864.0, + "64": 1877092864.0, + "65": 1877092864.0, + "66": 1877092864.0, + "67": 1877092864.0, + "68": 1877092864.0, + "69": 1877092864.0, + "70": 1877092864.0, + "71": 1877092864.0, + "72": 1877092864.0, + "73": 1877092864.0, + "74": 1877092864.0, + "75": 1877092864.0, + "76": 1877092864.0, + "77": 1877092864.0, + "78": 1877092864.0, + "79": 1877092864.0, + "80": 1877092864.0, + "81": 1877092864.0, + "82": 1877092864.0, + "83": 1877092864.0, + "84": 1877092864.0, + "85": 1877092864.0, + "86": 1877092864.0, + "87": 1877092864.0, + "88": 1877092864.0, + "89": 1877092864.0, + "90": 1877092864.0, + "91": 1877092864.0, + "92": 1877092864.0, + "93": 1877092864.0, + "94": 1877092864.0, + "95": 1877092864.0, + "96": 1877092864.0, + "97": 1877092864.0, + "98": 1877092864.0, + "99": 1877092864.0, + "100": 1877092864.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.19311, + "2": 0.24225, + "3": 0.2087, + "4": 0.20348, + "5": 0.20247, + "6": 0.20007, + "7": 0.19973, + "8": 0.19722, + "9": 0.19776, + "10": 0.20373, + "11": 0.20497, + "12": 0.19614, + "13": 0.19597, + "14": 0.1953, + "15": 0.19603, + "16": 0.19598, + "17": 0.19564, + "18": 0.19548, + "19": 0.19581, + "20": 0.19578, + "21": 0.20159, + "22": 0.204, + "23": 0.19784, + "24": 0.19568, + "25": 0.19673, + "26": 0.1956, + "27": 0.19646, + "28": 0.19626, + "29": 0.19657, + "30": 0.1964, + "31": 0.19757, + "32": 0.19691, + "33": 0.19767, + "34": 0.1964, + "35": 0.19629, + "36": 0.19689, + "37": 0.19658, + "38": 0.19802, + "39": 0.19595, + "40": 0.19606, + "41": 0.1981, + "42": 0.19694, + "43": 0.19674, + "44": 0.20226, + "45": 0.20006, + "46": 0.19944, + "47": 0.19874, + "48": 0.20091, + "49": 0.19983, + "50": 0.19769, + "51": 0.21244, + "52": 0.19803, + "53": 0.1979, + "54": 0.19796, + "55": 0.19769, + "56": 0.1993, + "57": 0.19707, + "58": 0.19811, + "59": 0.1965, + "60": 0.19805, + "61": 0.19725, + "62": 0.19705, + "63": 0.19795, + "64": 0.19775, + "65": 0.1969, + "66": 0.19789, + "67": 0.19776, + "68": 0.19742, + "69": 0.19762, + "70": 0.1978, + "71": 0.19703, + "72": 0.19789, + "73": 0.19833, + "74": 0.19789, + "75": 0.19748, + "76": 0.19777, + "77": 0.19741, + "78": 0.19731, + "79": 0.19842, + "80": 0.19795, + "81": 0.19717, + "82": 0.19778, + "83": 0.1986, + "84": 0.19775, + "85": 0.19774, + "86": 0.19712, + "87": 0.1982, + "88": 0.19654, + "89": 0.19714, + "90": 0.19748, + "91": 0.19697, + "92": 0.19735, + "93": 0.19818, + "94": 0.19742, + "95": 0.19796, + "96": 0.19716, + "97": 0.19766, + "98": 0.19931, + "99": 0.19795, + "100": 0.19749 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json new file mode 100644 index 00000000000..246a0565ee7 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.92228, + "2": 10.92833, + "3": 10.91713, + "4": 10.90495, + "5": 10.92807, + "6": 10.93673, + "7": 10.90406, + "8": 10.92229, + "9": 10.91254, + "10": 10.90852, + "11": 10.89335, + "12": 10.9208, + "13": 10.91496, + "14": 10.92145, + "15": 10.88435, + "16": 10.87455, + "17": 10.83916, + "18": 10.87309, + "19": 10.85325, + "20": 10.77493, + "21": 10.74754, + "22": 10.63148, + "23": 10.75622, + "24": 10.65569, + "25": 10.59216, + "26": 10.65327, + "27": 10.64876, + "28": 10.59656, + "29": 10.61011, + "30": 10.39286, + "31": 10.15725, + "32": 10.49217, + "33": 10.47941, + "34": 10.24014, + "35": 10.2971, + "36": 10.24566, + "37": 10.35283, + "38": 10.20534, + "39": 10.40418, + "40": 10.09553, + "41": 10.15279, + "42": 10.2188, + "43": 9.85527, + "44": 9.96244, + "45": 9.84613, + "46": 9.83799, + "47": 10.13884, + "48": 9.85689, + "49": 9.53747, + "50": 9.90876, + "51": 9.84971, + "52": 9.74156, + "53": 10.06322, + "54": 9.94581, + "55": 9.87731, + "56": 9.62746, + "57": 9.47259, + "58": 9.82912, + "59": 9.583, + "60": 9.49181, + "61": 9.69961, + "62": 9.98089, + "63": 9.37212, + "64": 9.7756, + "65": 8.9433, + "66": 9.69993, + "67": 9.36414, + "68": 9.78706, + "69": 9.78397, + "70": 9.72288, + "71": 9.60749, + "72": 9.58416, + "73": 9.49093, + "74": 8.94864, + "75": 9.41807, + "76": 9.08721, + "77": 10.06283, + "78": 9.729, + "79": 9.37091, + "80": 9.40033, + "81": 9.47754, + "82": 9.69121, + "83": 9.30762, + "84": 9.41252, + "85": 9.61132, + "86": 9.07621, + "87": 9.59459, + "88": 9.74768, + "89": 9.6068, + "90": 9.81078, + "91": 9.34441, + "92": 9.36535, + "93": 9.07743, + "94": 8.82975, + "95": 9.51676, + "96": 9.52546, + "97": 9.31031, + "98": 9.67812, + "99": 8.88848, + "100": 9.40128 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1681.0, + "2": 1822.0, + "3": 1757.0, + "4": 1751.0, + "5": 1937.0, + "6": 1886.0, + "7": 1837.0, + "8": 1786.0, + "9": 1935.0, + "10": 1542.0, + "11": 1896.0, + "12": 1808.0, + "13": 1960.0, + "14": 1846.0, + "15": 1948.0, + "16": 1903.0, + "17": 1900.0, + "18": 1725.0, + "19": 1812.0, + "20": 1647.0, + "21": 1886.0, + "22": 1675.0, + "23": 1992.0, + "24": 1621.0, + "25": 1624.0, + "26": 1778.0, + "27": 1925.0, + "28": 1996.0, + "29": 2074.0, + "30": 1899.0, + "31": 1539.0, + "32": 1956.0, + "33": 2254.0, + "34": 1927.0, + "35": 2029.0, + "36": 1986.0, + "37": 2377.0, + "38": 2234.0, + "39": 2396.0, + "40": 2123.0, + "41": 2316.0, + "42": 2245.0, + "43": 2077.0, + "44": 2179.0, + "45": 2078.0, + "46": 2280.0, + "47": 2573.0, + "48": 2440.0, + "49": 2213.0, + "50": 2532.0, + "51": 2735.0, + "52": 2607.0, + "53": 2951.0, + "54": 2672.0, + "55": 2451.0, + "56": 2712.0, + "57": 2392.0, + "58": 2979.0, + "59": 2869.0, + "60": 2435.0, + "61": 2938.0, + "62": 2669.0, + "63": 2392.0, + "64": 2998.0, + "65": 2689.0, + "66": 3285.0, + "67": 2782.0, + "68": 2753.0, + "69": 2958.0, + "70": 3271.0, + "71": 3040.0, + "72": 2504.0, + "73": 3096.0, + "74": 1910.0, + "75": 2617.0, + "76": 3081.0, + "77": 3390.0, + "78": 3186.0, + "79": 3320.0, + "80": 3483.0, + "81": 3782.0, + "82": 3516.0, + "83": 2864.0, + "84": 3396.0, + "85": 3247.0, + "86": 2785.0, + "87": 3762.0, + "88": 3102.0, + "89": 3483.0, + "90": 3076.0, + "91": 2643.0, + "92": 3198.0, + "93": 2666.0, + "94": 3390.0, + "95": 3410.0, + "96": 3508.0, + "97": 3178.0, + "98": 3865.0, + "99": 3143.0, + "100": 3357.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 482499072.0, + "2": 482499072.0, + "3": 482499072.0, + "4": 482499072.0, + "5": 482499072.0, + "6": 482499072.0, + "7": 482499072.0, + "8": 482499072.0, + "9": 482499072.0, + "10": 482499072.0, + "11": 482499072.0, + "12": 482499072.0, + "13": 482499072.0, + "14": 482499072.0, + "15": 482499072.0, + "16": 482499072.0, + "17": 482499072.0, + "18": 482499072.0, + "19": 482499072.0, + "20": 482499072.0, + "21": 482499072.0, + "22": 482499072.0, + "23": 482499072.0, + "24": 482499072.0, + "25": 482499072.0, + "26": 482499072.0, + "27": 482499072.0, + "28": 482499072.0, + "29": 482499072.0, + "30": 482499072.0, + "31": 482499072.0, + "32": 482499072.0, + "33": 482499072.0, + "34": 482499072.0, + "35": 482499072.0, + "36": 482499072.0, + "37": 482499072.0, + "38": 482499072.0, + "39": 482499072.0, + "40": 482499072.0, + "41": 482499072.0, + "42": 482499072.0, + "43": 482499072.0, + "44": 482499072.0, + "45": 482499072.0, + "46": 482499072.0, + "47": 482499072.0, + "48": 482499072.0, + "49": 482499072.0, + "50": 482499072.0, + "51": 482499072.0, + "52": 482499072.0, + "53": 482499072.0, + "54": 482499072.0, + "55": 482499072.0, + "56": 482499072.0, + "57": 482499072.0, + "58": 482499072.0, + "59": 482499072.0, + "60": 482499072.0, + "61": 482499072.0, + "62": 482499072.0, + "63": 482499072.0, + "64": 482499072.0, + "65": 482499072.0, + "66": 482499072.0, + "67": 482499072.0, + "68": 482499072.0, + "69": 482499072.0, + "70": 482499072.0, + "71": 482499072.0, + "72": 482499072.0, + "73": 482499072.0, + "74": 482499072.0, + "75": 482499072.0, + "76": 482499072.0, + "77": 482499072.0, + "78": 482499072.0, + "79": 482499072.0, + "80": 482499072.0, + "81": 482499072.0, + "82": 482499072.0, + "83": 482499072.0, + "84": 482499072.0, + "85": 482499072.0, + "86": 482499072.0, + "87": 482499072.0, + "88": 482499072.0, + "89": 482499072.0, + "90": 482499072.0, + "91": 482499072.0, + "92": 482499072.0, + "93": 482499072.0, + "94": 482499072.0, + "95": 482499072.0, + "96": 482499072.0, + "97": 482499072.0, + "98": 482499072.0, + "99": 482499072.0, + "100": 482499072.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1222004224.0, + "2": 1401029120.0, + "3": 1401029120.0, + "4": 1401029120.0, + "5": 1401029120.0, + "6": 1401029120.0, + "7": 1401029120.0, + "8": 1401029120.0, + "9": 1401029120.0, + "10": 1401029120.0, + "11": 1401029120.0, + "12": 1401029120.0, + "13": 1401029120.0, + "14": 1401029120.0, + "15": 1401029120.0, + "16": 1401029120.0, + "17": 1401029120.0, + "18": 1401029120.0, + "19": 1401029120.0, + "20": 1401029120.0, + "21": 1401029120.0, + "22": 1401029120.0, + "23": 1401029120.0, + "24": 1401029120.0, + "25": 1401029120.0, + "26": 1401029120.0, + "27": 1401029120.0, + "28": 1401029120.0, + "29": 1401029120.0, + "30": 1401029120.0, + "31": 1401029120.0, + "32": 1401029120.0, + "33": 1401029120.0, + "34": 1401029120.0, + "35": 1401029120.0, + "36": 1401029120.0, + "37": 1401029120.0, + "38": 1401029120.0, + "39": 1401029120.0, + "40": 1401029120.0, + "41": 1401029120.0, + "42": 1401029120.0, + "43": 1401029120.0, + "44": 1401029120.0, + "45": 1401029120.0, + "46": 1401029120.0, + "47": 1401029120.0, + "48": 1401029120.0, + "49": 1401029120.0, + "50": 1401029120.0, + "51": 1401029120.0, + "52": 1401029120.0, + "53": 1401029120.0, + "54": 1401029120.0, + "55": 1401029120.0, + "56": 1401029120.0, + "57": 1401029120.0, + "58": 1401029120.0, + "59": 1401029120.0, + "60": 1401029120.0, + "61": 1401029120.0, + "62": 1401029120.0, + "63": 1401029120.0, + "64": 1401029120.0, + "65": 1401029120.0, + "66": 1401029120.0, + "67": 1401029120.0, + "68": 1401029120.0, + "69": 1401029120.0, + "70": 1401029120.0, + "71": 1401029120.0, + "72": 1401029120.0, + "73": 1401029120.0, + "74": 1401029120.0, + "75": 1401029120.0, + "76": 1401029120.0, + "77": 1401029120.0, + "78": 1401029120.0, + "79": 1401029120.0, + "80": 1401029120.0, + "81": 1401029120.0, + "82": 1401029120.0, + "83": 1401029120.0, + "84": 1401029120.0, + "85": 1401029120.0, + "86": 1401029120.0, + "87": 1401029120.0, + "88": 1401029120.0, + "89": 1401029120.0, + "90": 1401029120.0, + "91": 1401029120.0, + "92": 1401029120.0, + "93": 1401029120.0, + "94": 1401029120.0, + "95": 1401029120.0, + "96": 1401029120.0, + "97": 1401029120.0, + "98": 1401029120.0, + "99": 1401029120.0, + "100": 1401029120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.96786, + "2": 0.22842, + "3": 0.19642, + "4": 0.19541, + "5": 0.19655, + "6": 0.19661, + "7": 0.1995, + "8": 0.19811, + "9": 0.20663, + "10": 0.19861, + "11": 0.19844, + "12": 0.19977, + "13": 0.19786, + "14": 0.20035, + "15": 0.1971, + "16": 0.19641, + "17": 0.19584, + "18": 0.19562, + "19": 0.19543, + "20": 0.1957, + "21": 0.19839, + "22": 0.19566, + "23": 0.19575, + "24": 0.1965, + "25": 0.19478, + "26": 0.19171, + "27": 0.19197, + "28": 0.1919, + "29": 0.19176, + "30": 0.19205, + "31": 0.1915, + "32": 0.19135, + "33": 0.19395, + "34": 0.19259, + "35": 0.19293, + "36": 0.19275, + "37": 0.19242, + "38": 0.1918, + "39": 0.19242, + "40": 0.19263, + "41": 0.19197, + "42": 0.19307, + "43": 0.19198, + "44": 0.19173, + "45": 0.19326, + "46": 0.19327, + "47": 0.19349, + "48": 0.19306, + "49": 0.19232, + "50": 0.19187, + "51": 0.20758, + "52": 0.1977, + "53": 0.19793, + "54": 0.1985, + "55": 0.19827, + "56": 0.19849, + "57": 0.19828, + "58": 0.19767, + "59": 0.19786, + "60": 0.19794, + "61": 0.19844, + "62": 0.19936, + "63": 0.19766, + "64": 0.19773, + "65": 0.19892, + "66": 0.19913, + "67": 0.19939, + "68": 0.19753, + "69": 0.19713, + "70": 0.19916, + "71": 0.19721, + "72": 0.19768, + "73": 0.19805, + "74": 0.19738, + "75": 0.19678, + "76": 0.19752, + "77": 0.19851, + "78": 0.1964, + "79": 0.19778, + "80": 0.19767, + "81": 0.19811, + "82": 0.19926, + "83": 0.19863, + "84": 0.19867, + "85": 0.19775, + "86": 0.19856, + "87": 0.19932, + "88": 0.19833, + "89": 0.19798, + "90": 0.19896, + "91": 0.19713, + "92": 0.19767, + "93": 0.19793, + "94": 0.1989, + "95": 0.19691, + "96": 0.1975, + "97": 0.19677, + "98": 0.1982, + "99": 0.19568, + "100": 0.19451 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..0759a3ad35e --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.85961, + "2": 10.85774, + "3": 10.85857, + "4": 10.84879, + "5": 10.88247, + "6": 10.88701, + "7": 10.86152, + "8": 10.86792, + "9": 10.86512, + "10": 10.83742, + "11": 10.87758, + "12": 10.87232, + "13": 10.87849, + "14": 10.88848, + "15": 10.82866, + "16": 10.8307, + "17": 10.79817, + "18": 10.82417, + "19": 10.81181, + "20": 10.72691, + "21": 10.7183, + "22": 10.57059, + "23": 10.72817, + "24": 10.61403, + "25": 10.55842, + "26": 10.60676, + "27": 10.61661, + "28": 10.57482, + "29": 10.59587, + "30": 10.37815, + "31": 10.13027, + "32": 10.47778, + "33": 10.46854, + "34": 10.22658, + "35": 10.27984, + "36": 10.22842, + "37": 10.35309, + "38": 10.20319, + "39": 10.4096, + "40": 10.0888, + "41": 10.15947, + "42": 10.2147, + "43": 9.84971, + "44": 9.96501, + "45": 9.82545, + "46": 9.83484, + "47": 10.15515, + "48": 9.84462, + "49": 9.52863, + "50": 9.91253 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1816.0, + "2": 1884.0, + "3": 1906.0, + "4": 1901.0, + "5": 2061.0, + "6": 2186.0, + "7": 2145.0, + "8": 1775.0, + "9": 2106.0, + "10": 1630.0, + "11": 2236.0, + "12": 2015.0, + "13": 2168.0, + "14": 2021.0, + "15": 2088.0, + "16": 2152.0, + "17": 1976.0, + "18": 1952.0, + "19": 2046.0, + "20": 1740.0, + "21": 2073.0, + "22": 1810.0, + "23": 2186.0, + "24": 1839.0, + "25": 1681.0, + "26": 1962.0, + "27": 2009.0, + "28": 2204.0, + "29": 2264.0, + "30": 2117.0, + "31": 1873.0, + "32": 2178.0, + "33": 2372.0, + "34": 2127.0, + "35": 2225.0, + "36": 2140.0, + "37": 2649.0, + "38": 2602.0, + "39": 2607.0, + "40": 2574.0, + "41": 2503.0, + "42": 2532.0, + "43": 2285.0, + "44": 2313.0, + "45": 2423.0, + "46": 2563.0, + "47": 2856.0, + "48": 2629.0, + "49": 2566.0, + "50": 2651.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1449634304.0, + "2": 1516021248.0, + "3": 1516021248.0, + "4": 1516021248.0, + "5": 1516021248.0, + "6": 1516021248.0, + "7": 1516021248.0, + "8": 1516021248.0, + "9": 1516021248.0, + "10": 1516021248.0, + "11": 1516021248.0, + "12": 1516021248.0, + "13": 1516021248.0, + "14": 1516021248.0, + "15": 1516021248.0, + "16": 1516021248.0, + "17": 1516021248.0, + "18": 1516021248.0, + "19": 1516021248.0, + "20": 1516021248.0, + "21": 1516021248.0, + "22": 1516021248.0, + "23": 1516021248.0, + "24": 1516021248.0, + "25": 1516021248.0, + "26": 1516021248.0, + "27": 1516021248.0, + "28": 1516021248.0, + "29": 1516021248.0, + "30": 1516021248.0, + "31": 1516021248.0, + "32": 1516021248.0, + "33": 1516021248.0, + "34": 1516021248.0, + "35": 1516021248.0, + "36": 1516021248.0, + "37": 1516021248.0, + "38": 1516021248.0, + "39": 1516021248.0, + "40": 1516021248.0, + "41": 1516021248.0, + "42": 1516021248.0, + "43": 1516021248.0, + "44": 1516021248.0, + "45": 1516021248.0, + "46": 1516021248.0, + "47": 1516021248.0, + "48": 1516021248.0, + "49": 1516021248.0, + "50": 1516021248.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.53093, + "2": 0.38997, + "3": 0.33818, + "4": 0.33571, + "5": 0.33595, + "6": 0.33847, + "7": 0.3396, + "8": 0.34055, + "9": 0.33978, + "10": 0.33979, + "11": 0.33823, + "12": 0.33867, + "13": 0.33452, + "14": 0.32817, + "15": 0.32937, + "16": 0.32801, + "17": 0.32712, + "18": 0.32949, + "19": 0.32671, + "20": 0.3289, + "21": 0.3281, + "22": 0.32727, + "23": 0.32856, + "24": 0.32695, + "25": 0.32632, + "26": 0.32717, + "27": 0.32698, + "28": 0.32772, + "29": 0.32669, + "30": 0.32925, + "31": 0.3264, + "32": 0.32896, + "33": 0.32706, + "34": 0.32907, + "35": 0.34059, + "36": 0.34365, + "37": 0.33146, + "38": 0.32478, + "39": 0.3305, + "40": 0.32726, + "41": 0.32743, + "42": 0.32937, + "43": 0.32879, + "44": 0.32853, + "45": 0.32907, + "46": 0.32817, + "47": 0.32958, + "48": 0.332, + "49": 0.33128, + "50": 0.32696 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json new file mode 100644 index 00000000000..df2a22bb193 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971619840.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.69956, + "2": 0.34443, + "3": 0.29026, + "4": 0.2891, + "5": 0.28974, + "6": 0.29236, + "7": 0.29315, + "8": 0.29061, + "9": 0.28978, + "10": 0.29069, + "11": 0.28686, + "12": 0.2841, + "13": 0.28347, + "14": 0.28381, + "15": 0.28327, + "16": 0.28386, + "17": 0.28376, + "18": 0.28595, + "19": 0.28302, + "20": 0.28263, + "21": 0.28413, + "22": 0.28377, + "23": 0.28327, + "24": 0.28362, + "25": 0.28297, + "26": 0.28635, + "27": 0.28461, + "28": 0.28549, + "29": 0.28446, + "30": 0.28309, + "31": 0.28399, + "32": 0.28389, + "33": 0.28395, + "34": 0.2838, + "35": 0.2837, + "36": 0.28439, + "37": 0.287, + "38": 0.28462, + "39": 0.28334, + "40": 0.28407, + "41": 0.28402, + "42": 0.28351, + "43": 0.28318, + "44": 0.28387, + "45": 0.28352, + "46": 0.29226, + "47": 0.28366, + "48": 0.2851, + "49": 0.28439, + "50": 0.28412, + "51": 0.29357, + "52": 0.28925, + "53": 0.28952, + "54": 0.28895, + "55": 0.29016, + "56": 0.28838, + "57": 0.28921, + "58": 0.2895, + "59": 0.28925, + "60": 0.28911, + "61": 0.28861, + "62": 0.28854, + "63": 0.28837, + "64": 0.28828, + "65": 0.28725, + "66": 0.28793, + "67": 0.28849, + "68": 0.28819, + "69": 0.28835, + "70": 0.29026, + "71": 0.29094, + "72": 0.28885, + "73": 0.2901, + "74": 0.28981, + "75": 0.29022, + "76": 0.28993, + "77": 0.29007, + "78": 0.2898, + "79": 0.28878, + "80": 0.29006, + "81": 0.28956, + "82": 0.28972, + "83": 0.28973, + "84": 0.28997, + "85": 0.28876, + "86": 0.28891, + "87": 0.28829, + "88": 0.28939, + "89": 0.29039, + "90": 0.28933, + "91": 0.28991, + "92": 0.2901, + "93": 0.29016, + "94": 0.29013, + "95": 0.28816, + "96": 0.28826, + "97": 0.28825, + "98": 0.28987, + "99": 0.28901, + "100": 0.28895 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..d156d33c303 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85932, + "2": 10.85709, + "3": 10.85896, + "4": 10.84972, + "5": 10.88254, + "6": 10.88712, + "7": 10.86187, + "8": 10.86848, + "9": 10.86474, + "10": 10.83743, + "11": 10.87761, + "12": 10.87275, + "13": 10.87805, + "14": 10.88829, + "15": 10.8292, + "16": 10.83093, + "17": 10.79858, + "18": 10.82391, + "19": 10.8112, + "20": 10.72714, + "21": 10.71846, + "22": 10.57021, + "23": 10.72839, + "24": 10.6137, + "25": 10.55895, + "26": 10.60667, + "27": 10.61746, + "28": 10.57512, + "29": 10.59634, + "30": 10.37833, + "31": 10.13006, + "32": 10.47788, + "33": 10.46827, + "34": 10.22647, + "35": 10.28047, + "36": 10.22818, + "37": 10.35344, + "38": 10.203, + "39": 10.40996, + "40": 10.08874, + "41": 10.15951, + "42": 10.2151, + "43": 9.84978, + "44": 9.96524, + "45": 9.82532, + "46": 9.83508, + "47": 10.15501, + "48": 9.8447, + "49": 9.5289, + "50": 9.91268, + "51": 9.85065, + "52": 9.7464, + "53": 10.07271, + "54": 9.95757, + "55": 9.87725, + "56": 9.62951, + "57": 9.48816, + "58": 9.83239, + "59": 9.58985, + "60": 9.50827, + "61": 9.6947, + "62": 9.99304, + "63": 9.37511, + "64": 9.77996, + "65": 8.95215, + "66": 9.71323, + "67": 9.37884, + "68": 9.78794, + "69": 9.79078, + "70": 9.7308, + "71": 9.61793, + "72": 9.59094, + "73": 9.49435, + "74": 8.94865, + "75": 9.43606, + "76": 9.09894, + "77": 10.06437, + "78": 9.73006, + "79": 9.37771, + "80": 9.41266, + "81": 9.4854, + "82": 9.69576, + "83": 9.32017, + "84": 9.42235, + "85": 9.61578, + "86": 9.07218, + "87": 9.59328, + "88": 9.7509, + "89": 9.61159, + "90": 9.82148, + "91": 9.35304, + "92": 9.36254, + "93": 9.08747, + "94": 8.83398, + "95": 9.51923, + "96": 9.52595, + "97": 9.31413, + "98": 9.67414, + "99": 8.88869, + "100": 9.40651 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1754.0, + "2": 1848.0, + "3": 1845.0, + "4": 1871.0, + "5": 2173.0, + "6": 2054.0, + "7": 2021.0, + "8": 1737.0, + "9": 2044.0, + "10": 1524.0, + "11": 2198.0, + "12": 2030.0, + "13": 2211.0, + "14": 2094.0, + "15": 2036.0, + "16": 2165.0, + "17": 2044.0, + "18": 1842.0, + "19": 2002.0, + "20": 1774.0, + "21": 1976.0, + "22": 1869.0, + "23": 2232.0, + "24": 1769.0, + "25": 1714.0, + "26": 1866.0, + "27": 2106.0, + "28": 2287.0, + "29": 2221.0, + "30": 1967.0, + "31": 1788.0, + "32": 2187.0, + "33": 2432.0, + "34": 2059.0, + "35": 2308.0, + "36": 2173.0, + "37": 2619.0, + "38": 2572.0, + "39": 2618.0, + "40": 2548.0, + "41": 2531.0, + "42": 2447.0, + "43": 2283.0, + "44": 2358.0, + "45": 2398.0, + "46": 2572.0, + "47": 2818.0, + "48": 2599.0, + "49": 2579.0, + "50": 2731.0, + "51": 2873.0, + "52": 2946.0, + "53": 3158.0, + "54": 2907.0, + "55": 2740.0, + "56": 3029.0, + "57": 2489.0, + "58": 3327.0, + "59": 3042.0, + "60": 2780.0, + "61": 3302.0, + "62": 2961.0, + "63": 2702.0, + "64": 3318.0, + "65": 2909.0, + "66": 3513.0, + "67": 2959.0, + "68": 2963.0, + "69": 3171.0, + "70": 3547.0, + "71": 3246.0, + "72": 2586.0, + "73": 3301.0, + "74": 2135.0, + "75": 2752.0, + "76": 3275.0, + "77": 3648.0, + "78": 3472.0, + "79": 3536.0, + "80": 3685.0, + "81": 4159.0, + "82": 3488.0, + "83": 3179.0, + "84": 3639.0, + "85": 3631.0, + "86": 3045.0, + "87": 4315.0, + "88": 3481.0, + "89": 3819.0, + "90": 3323.0, + "91": 3014.0, + "92": 3581.0, + "93": 2932.0, + "94": 3715.0, + "95": 3593.0, + "96": 3764.0, + "97": 3582.0, + "98": 3998.0, + "99": 3406.0, + "100": 3521.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 269842944.0, + "2": 269842944.0, + "3": 269842944.0, + "4": 269842944.0, + "5": 269842944.0, + "6": 269842944.0, + "7": 269842944.0, + "8": 269842944.0, + "9": 269842944.0, + "10": 269842944.0, + "11": 269842944.0, + "12": 269842944.0, + "13": 269842944.0, + "14": 269842944.0, + "15": 269842944.0, + "16": 269842944.0, + "17": 269842944.0, + "18": 269842944.0, + "19": 269842944.0, + "20": 269842944.0, + "21": 269842944.0, + "22": 269842944.0, + "23": 269842944.0, + "24": 269842944.0, + "25": 269842944.0, + "26": 269842944.0, + "27": 269842944.0, + "28": 269842944.0, + "29": 269842944.0, + "30": 269842944.0, + "31": 269842944.0, + "32": 269842944.0, + "33": 269842944.0, + "34": 269842944.0, + "35": 269842944.0, + "36": 269842944.0, + "37": 269842944.0, + "38": 269842944.0, + "39": 269842944.0, + "40": 269842944.0, + "41": 269842944.0, + "42": 269842944.0, + "43": 269842944.0, + "44": 269842944.0, + "45": 269842944.0, + "46": 269842944.0, + "47": 269842944.0, + "48": 269842944.0, + "49": 269842944.0, + "50": 269842944.0, + "51": 269842944.0, + "52": 269842944.0, + "53": 269842944.0, + "54": 269842944.0, + "55": 269842944.0, + "56": 269842944.0, + "57": 269842944.0, + "58": 269842944.0, + "59": 269842944.0, + "60": 269842944.0, + "61": 269842944.0, + "62": 269842944.0, + "63": 269842944.0, + "64": 269842944.0, + "65": 269842944.0, + "66": 269842944.0, + "67": 269842944.0, + "68": 269842944.0, + "69": 269842944.0, + "70": 269842944.0, + "71": 269842944.0, + "72": 269842944.0, + "73": 269842944.0, + "74": 269842944.0, + "75": 269842944.0, + "76": 269842944.0, + "77": 269842944.0, + "78": 269842944.0, + "79": 269842944.0, + "80": 269842944.0, + "81": 269842944.0, + "82": 269842944.0, + "83": 269842944.0, + "84": 269842944.0, + "85": 269842944.0, + "86": 269842944.0, + "87": 269842944.0, + "88": 269842944.0, + "89": 269842944.0, + "90": 269842944.0, + "91": 269842944.0, + "92": 269842944.0, + "93": 269842944.0, + "94": 269842944.0, + "95": 269842944.0, + "96": 269842944.0, + "97": 269842944.0, + "98": 269842944.0, + "99": 269842944.0, + "100": 269842944.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 971620864.0, + "2": 1036172800.0, + "3": 1036172800.0, + "4": 1036172800.0, + "5": 1036172800.0, + "6": 1036172800.0, + "7": 1036172800.0, + "8": 1036172800.0, + "9": 1036172800.0, + "10": 1036172800.0, + "11": 1036172800.0, + "12": 1036172800.0, + "13": 1036172800.0, + "14": 1036172800.0, + "15": 1036172800.0, + "16": 1036172800.0, + "17": 1036172800.0, + "18": 1036172800.0, + "19": 1036172800.0, + "20": 1036172800.0, + "21": 1036172800.0, + "22": 1036172800.0, + "23": 1036172800.0, + "24": 1036172800.0, + "25": 1036172800.0, + "26": 1036172800.0, + "27": 1036172800.0, + "28": 1036172800.0, + "29": 1036172800.0, + "30": 1036172800.0, + "31": 1036172800.0, + "32": 1036172800.0, + "33": 1036172800.0, + "34": 1036172800.0, + "35": 1036172800.0, + "36": 1036172800.0, + "37": 1036172800.0, + "38": 1036172800.0, + "39": 1036172800.0, + "40": 1036172800.0, + "41": 1036172800.0, + "42": 1036172800.0, + "43": 1036172800.0, + "44": 1036172800.0, + "45": 1036172800.0, + "46": 1036172800.0, + "47": 1036172800.0, + "48": 1036172800.0, + "49": 1036172800.0, + "50": 1036172800.0, + "51": 1036172800.0, + "52": 1036172800.0, + "53": 1036172800.0, + "54": 1036172800.0, + "55": 1036172800.0, + "56": 1036172800.0, + "57": 1036172800.0, + "58": 1036172800.0, + "59": 1036172800.0, + "60": 1036172800.0, + "61": 1036172800.0, + "62": 1036172800.0, + "63": 1036172800.0, + "64": 1036172800.0, + "65": 1036172800.0, + "66": 1036172800.0, + "67": 1036172800.0, + "68": 1036172800.0, + "69": 1036172800.0, + "70": 1036172800.0, + "71": 1036172800.0, + "72": 1036172800.0, + "73": 1036172800.0, + "74": 1036172800.0, + "75": 1036172800.0, + "76": 1036172800.0, + "77": 1036172800.0, + "78": 1036172800.0, + "79": 1036172800.0, + "80": 1036172800.0, + "81": 1036172800.0, + "82": 1036172800.0, + "83": 1036172800.0, + "84": 1036172800.0, + "85": 1036172800.0, + "86": 1036172800.0, + "87": 1036172800.0, + "88": 1036172800.0, + "89": 1036172800.0, + "90": 1036172800.0, + "91": 1036172800.0, + "92": 1036172800.0, + "93": 1036172800.0, + "94": 1036172800.0, + "95": 1036172800.0, + "96": 1036172800.0, + "97": 1036172800.0, + "98": 1036172800.0, + "99": 1036172800.0, + "100": 1036172800.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.27545, + "2": 0.37218, + "3": 0.29526, + "4": 0.29243, + "5": 0.28826, + "6": 0.29166, + "7": 0.29473, + "8": 0.29416, + "9": 0.29471, + "10": 0.29535, + "11": 0.29549, + "12": 0.2972, + "13": 0.29831, + "14": 0.2948, + "15": 0.29657, + "16": 0.29666, + "17": 0.29485, + "18": 0.30158, + "19": 0.29495, + "20": 0.29383, + "21": 0.29583, + "22": 0.29318, + "23": 0.29279, + "24": 0.29527, + "25": 0.29573, + "26": 0.29431, + "27": 0.29367, + "28": 0.29625, + "29": 0.2982, + "30": 0.29548, + "31": 0.29582, + "32": 0.29503, + "33": 0.29378, + "34": 0.29457, + "35": 0.29586, + "36": 0.29322, + "37": 0.29462, + "38": 0.29411, + "39": 0.29285, + "40": 0.29626, + "41": 0.29395, + "42": 0.29359, + "43": 0.29317, + "44": 0.293, + "45": 0.29298, + "46": 0.29325, + "47": 0.29576, + "48": 0.29557, + "49": 0.2936, + "50": 0.29339, + "51": 0.29808, + "52": 0.29361, + "53": 0.29487, + "54": 0.2948, + "55": 0.2935, + "56": 0.29416, + "57": 0.29433, + "58": 0.29294, + "59": 0.29404, + "60": 0.29409, + "61": 0.29306, + "62": 0.29269, + "63": 0.29337, + "64": 0.29412, + "65": 0.29357, + "66": 0.2938, + "67": 0.29321, + "68": 0.29336, + "69": 0.29304, + "70": 0.29279, + "71": 0.29309, + "72": 0.29365, + "73": 0.29409, + "74": 0.53313, + "75": 0.29324, + "76": 0.2929, + "77": 0.29289, + "78": 0.29229, + "79": 0.29336, + "80": 0.29242, + "81": 0.29375, + "82": 0.29406, + "83": 0.29464, + "84": 0.29476, + "85": 0.29491, + "86": 0.29298, + "87": 0.29278, + "88": 0.29326, + "89": 0.29204, + "90": 0.29677, + "91": 0.29302, + "92": 0.29428, + "93": 0.30076, + "94": 0.29286, + "95": 0.29206, + "96": 0.29236, + "97": 0.29253, + "98": 0.29432, + "99": 0.29486, + "100": 0.29315 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json new file mode 100644 index 00000000000..3d184c491ba --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.85966, + "2": 10.85779, + "3": 10.85806, + "4": 10.85111, + "5": 10.88314, + "6": 10.8873, + "7": 10.86205, + "8": 10.86979, + "9": 10.86489, + "10": 10.83833, + "11": 10.87881, + "12": 10.87319, + "13": 10.87924, + "14": 10.88927, + "15": 10.83002, + "16": 10.83231, + "17": 10.80059, + "18": 10.82546, + "19": 10.81132, + "20": 10.73125, + "21": 10.72292, + "22": 10.57559, + "23": 10.73211, + "24": 10.61836, + "25": 10.56602, + "26": 10.61211, + "27": 10.62224, + "28": 10.58011, + "29": 10.59905, + "30": 10.38311, + "31": 10.14181, + "32": 10.48118, + "33": 10.47322, + "34": 10.23556, + "35": 10.28738, + "36": 10.23493, + "37": 10.35785, + "38": 10.20631, + "39": 10.41397, + "40": 10.09217, + "41": 10.16577, + "42": 10.21856, + "43": 9.85974, + "44": 9.97115, + "45": 9.83404, + "46": 9.84483, + "47": 10.16033, + "48": 9.85135, + "49": 9.53781, + "50": 9.91577, + "51": 9.8567, + "52": 9.75178, + "53": 10.07652, + "54": 9.96084, + "55": 9.88221, + "56": 9.63206, + "57": 9.49147, + "58": 9.83408, + "59": 9.59352, + "60": 9.51388, + "61": 9.69802, + "62": 9.99154, + "63": 9.3723, + "64": 9.77839, + "65": 8.95518, + "66": 9.70976, + "67": 9.38198, + "68": 9.78701, + "69": 9.793, + "70": 9.73033, + "71": 9.61752, + "72": 9.58459, + "73": 9.48958, + "74": 8.94015, + "75": 9.43092, + "76": 9.09168, + "77": 10.06222, + "78": 9.72696, + "79": 9.37408, + "80": 9.40676, + "81": 9.47995, + "82": 9.69225, + "83": 9.31299, + "84": 9.41921, + "85": 9.61096, + "86": 9.06853, + "87": 9.59119, + "88": 9.74582, + "89": 9.60624, + "90": 9.81746, + "91": 9.34247, + "92": 9.35856, + "93": 9.07894, + "94": 8.82753, + "95": 9.51606, + "96": 9.52063, + "97": 9.31097, + "98": 9.67055, + "99": 8.88626, + "100": 9.40485 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1756.0, + "2": 2002.0, + "3": 1950.0, + "4": 1894.0, + "5": 2227.0, + "6": 2168.0, + "7": 2021.0, + "8": 1775.0, + "9": 2051.0, + "10": 1611.0, + "11": 2147.0, + "12": 1965.0, + "13": 2185.0, + "14": 2052.0, + "15": 2134.0, + "16": 2077.0, + "17": 2080.0, + "18": 1914.0, + "19": 1993.0, + "20": 1823.0, + "21": 2142.0, + "22": 1847.0, + "23": 2340.0, + "24": 1831.0, + "25": 1790.0, + "26": 1923.0, + "27": 2066.0, + "28": 2268.0, + "29": 2261.0, + "30": 2043.0, + "31": 1759.0, + "32": 2122.0, + "33": 2389.0, + "34": 2186.0, + "35": 2312.0, + "36": 2176.0, + "37": 2626.0, + "38": 2471.0, + "39": 2624.0, + "40": 2556.0, + "41": 2475.0, + "42": 2504.0, + "43": 2326.0, + "44": 2307.0, + "45": 2355.0, + "46": 2625.0, + "47": 2809.0, + "48": 2621.0, + "49": 2670.0, + "50": 2699.0, + "51": 2829.0, + "52": 2809.0, + "53": 3230.0, + "54": 2864.0, + "55": 2706.0, + "56": 2917.0, + "57": 2529.0, + "58": 3339.0, + "59": 3051.0, + "60": 2623.0, + "61": 3287.0, + "62": 2913.0, + "63": 2639.0, + "64": 3154.0, + "65": 2856.0, + "66": 3465.0, + "67": 2934.0, + "68": 2985.0, + "69": 3298.0, + "70": 3653.0, + "71": 3260.0, + "72": 2684.0, + "73": 3232.0, + "74": 2191.0, + "75": 2766.0, + "76": 3335.0, + "77": 3793.0, + "78": 3608.0, + "79": 3384.0, + "80": 3782.0, + "81": 3969.0, + "82": 3640.0, + "83": 3237.0, + "84": 3606.0, + "85": 3553.0, + "86": 3160.0, + "87": 4130.0, + "88": 3430.0, + "89": 3818.0, + "90": 3363.0, + "91": 3041.0, + "92": 3524.0, + "93": 3060.0, + "94": 3575.0, + "95": 3463.0, + "96": 3921.0, + "97": 3597.0, + "98": 4039.0, + "99": 3435.0, + "100": 3548.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 368387584.0, + "2": 368387584.0, + "3": 368387584.0, + "4": 368387584.0, + "5": 368387584.0, + "6": 368387584.0, + "7": 368387584.0, + "8": 368387584.0, + "9": 368387584.0, + "10": 368387584.0, + "11": 368387584.0, + "12": 368387584.0, + "13": 368387584.0, + "14": 368387584.0, + "15": 368387584.0, + "16": 368387584.0, + "17": 368387584.0, + "18": 368387584.0, + "19": 368387584.0, + "20": 368387584.0, + "21": 368387584.0, + "22": 368387584.0, + "23": 368387584.0, + "24": 368387584.0, + "25": 368387584.0, + "26": 368387584.0, + "27": 368387584.0, + "28": 368387584.0, + "29": 368387584.0, + "30": 368387584.0, + "31": 368387584.0, + "32": 368387584.0, + "33": 368387584.0, + "34": 368387584.0, + "35": 368387584.0, + "36": 368387584.0, + "37": 368387584.0, + "38": 368387584.0, + "39": 368387584.0, + "40": 368387584.0, + "41": 368387584.0, + "42": 368387584.0, + "43": 368387584.0, + "44": 368387584.0, + "45": 368387584.0, + "46": 368387584.0, + "47": 368387584.0, + "48": 368387584.0, + "49": 368387584.0, + "50": 368387584.0, + "51": 368387584.0, + "52": 368387584.0, + "53": 368387584.0, + "54": 368387584.0, + "55": 368387584.0, + "56": 368387584.0, + "57": 368387584.0, + "58": 368387584.0, + "59": 368387584.0, + "60": 368387584.0, + "61": 368387584.0, + "62": 368387584.0, + "63": 368387584.0, + "64": 368387584.0, + "65": 368387584.0, + "66": 368387584.0, + "67": 368387584.0, + "68": 368387584.0, + "69": 368387584.0, + "70": 368387584.0, + "71": 368387584.0, + "72": 368387584.0, + "73": 368387584.0, + "74": 368387584.0, + "75": 368387584.0, + "76": 368387584.0, + "77": 368387584.0, + "78": 368387584.0, + "79": 368387584.0, + "80": 368387584.0, + "81": 368387584.0, + "82": 368387584.0, + "83": 368387584.0, + "84": 368387584.0, + "85": 368387584.0, + "86": 368387584.0, + "87": 368387584.0, + "88": 368387584.0, + "89": 368387584.0, + "90": 368387584.0, + "91": 368387584.0, + "92": 368387584.0, + "93": 368387584.0, + "94": 368387584.0, + "95": 368387584.0, + "96": 368387584.0, + "97": 368387584.0, + "98": 368387584.0, + "99": 368387584.0, + "100": 368387584.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1030075904.0, + "2": 1161062912.0, + "3": 1161062912.0, + "4": 1161062912.0, + "5": 1161062912.0, + "6": 1161062912.0, + "7": 1161062912.0, + "8": 1161062912.0, + "9": 1161062912.0, + "10": 1161062912.0, + "11": 1161062912.0, + "12": 1161062912.0, + "13": 1161062912.0, + "14": 1161062912.0, + "15": 1161062912.0, + "16": 1161062912.0, + "17": 1161062912.0, + "18": 1161062912.0, + "19": 1161062912.0, + "20": 1161062912.0, + "21": 1161062912.0, + "22": 1161062912.0, + "23": 1161062912.0, + "24": 1161062912.0, + "25": 1161062912.0, + "26": 1161062912.0, + "27": 1161062912.0, + "28": 1161062912.0, + "29": 1161062912.0, + "30": 1161062912.0, + "31": 1161062912.0, + "32": 1161062912.0, + "33": 1161062912.0, + "34": 1161062912.0, + "35": 1161062912.0, + "36": 1161062912.0, + "37": 1161062912.0, + "38": 1161062912.0, + "39": 1161062912.0, + "40": 1161062912.0, + "41": 1161062912.0, + "42": 1161062912.0, + "43": 1161062912.0, + "44": 1161062912.0, + "45": 1161062912.0, + "46": 1161062912.0, + "47": 1161062912.0, + "48": 1161062912.0, + "49": 1161062912.0, + "50": 1161062912.0, + "51": 1161062912.0, + "52": 1161062912.0, + "53": 1161062912.0, + "54": 1161062912.0, + "55": 1161062912.0, + "56": 1161062912.0, + "57": 1161062912.0, + "58": 1161062912.0, + "59": 1161062912.0, + "60": 1161062912.0, + "61": 1161062912.0, + "62": 1161062912.0, + "63": 1161062912.0, + "64": 1161062912.0, + "65": 1161062912.0, + "66": 1161062912.0, + "67": 1161062912.0, + "68": 1161062912.0, + "69": 1161062912.0, + "70": 1161062912.0, + "71": 1161062912.0, + "72": 1161062912.0, + "73": 1161062912.0, + "74": 1161062912.0, + "75": 1161062912.0, + "76": 1161062912.0, + "77": 1161062912.0, + "78": 1161062912.0, + "79": 1161062912.0, + "80": 1161062912.0, + "81": 1161062912.0, + "82": 1161062912.0, + "83": 1161062912.0, + "84": 1161062912.0, + "85": 1161062912.0, + "86": 1161062912.0, + "87": 1161062912.0, + "88": 1161062912.0, + "89": 1161062912.0, + "90": 1161062912.0, + "91": 1161062912.0, + "92": 1161062912.0, + "93": 1161062912.0, + "94": 1161062912.0, + "95": 1161062912.0, + "96": 1161062912.0, + "97": 1161062912.0, + "98": 1161062912.0, + "99": 1161062912.0, + "100": 1161062912.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.91324, + "2": 0.5374, + "3": 0.42257, + "4": 0.41964, + "5": 0.42006, + "6": 0.42252, + "7": 0.42386, + "8": 0.4225, + "9": 0.41894, + "10": 0.41907, + "11": 0.42057, + "12": 0.42029, + "13": 0.42305, + "14": 0.42046, + "15": 0.42057, + "16": 0.42121, + "17": 0.42211, + "18": 0.42182, + "19": 0.4229, + "20": 0.42063, + "21": 0.41956, + "22": 0.42035, + "23": 0.4214, + "24": 0.42135, + "25": 0.42124, + "26": 0.42145, + "27": 0.42052, + "28": 0.42091, + "29": 0.42047, + "30": 0.42247, + "31": 0.41604, + "32": 0.41597, + "33": 0.4159, + "34": 0.41717, + "35": 0.4166, + "36": 0.41726, + "37": 0.41724, + "38": 0.41918, + "39": 0.41675, + "40": 0.416, + "41": 0.41673, + "42": 0.41671, + "43": 0.41792, + "44": 0.41756, + "45": 0.41642, + "46": 0.41587, + "47": 0.41743, + "48": 0.41732, + "49": 0.41765, + "50": 0.41813, + "51": 0.41663, + "52": 0.41174, + "53": 0.40912, + "54": 0.40766, + "55": 0.40921, + "56": 0.40761, + "57": 0.4103, + "58": 0.4079, + "59": 0.80658, + "60": 0.40912, + "61": 0.40932, + "62": 0.41037, + "63": 0.41113, + "64": 0.4107, + "65": 0.40902, + "66": 0.40963, + "67": 0.40966, + "68": 0.40851, + "69": 0.41024, + "70": 0.40856, + "71": 0.40849, + "72": 0.40839, + "73": 0.40874, + "74": 0.40954, + "75": 0.40959, + "76": 0.40961, + "77": 0.41065, + "78": 0.40876, + "79": 0.41117, + "80": 0.41049, + "81": 0.41145, + "82": 0.4097, + "83": 0.40916, + "84": 0.40919, + "85": 0.40812, + "86": 0.80846, + "87": 0.40896, + "88": 0.41026, + "89": 0.40906, + "90": 0.40927, + "91": 0.41006, + "92": 0.41148, + "93": 0.40972, + "94": 0.41011, + "95": 0.40993, + "96": 0.40921, + "97": 0.41099, + "98": 0.40963, + "99": 0.40811, + "100": 0.40967 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json new file mode 100644 index 00000000000..3503b0f7133 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.84082, + "2": 10.83544, + "3": 10.83361, + "4": 10.8202, + "5": 10.84212, + "6": 10.87215, + "7": 10.83489, + "8": 10.8434, + "9": 10.84794, + "10": 10.81681, + "11": 10.86063, + "12": 10.85821, + "13": 10.87957, + "14": 10.87773, + "15": 10.85163, + "16": 10.8472, + "17": 10.84071, + "18": 10.86582, + "19": 10.85937, + "20": 10.85131, + "21": 10.85552, + "22": 10.82535, + "23": 10.86895, + "24": 10.82877, + "25": 10.81945, + "26": 10.83385, + "27": 10.82583, + "28": 10.84435, + "29": 10.83124, + "30": 10.75388, + "31": 10.66502, + "32": 10.78819, + "33": 10.76332, + "34": 10.67654, + "35": 10.68319, + "36": 10.63342, + "37": 10.68482, + "38": 10.60112, + "39": 10.69463, + "40": 10.52433, + "41": 10.54208, + "42": 10.56578, + "43": 10.34551, + "44": 10.38873, + "45": 10.31101, + "46": 10.30065, + "47": 10.48062, + "48": 10.28105, + "49": 10.05975, + "50": 10.29413, + "51": 10.23775, + "52": 10.15443, + "53": 10.36085, + "54": 10.26927, + "55": 10.2161, + "56": 9.99594, + "57": 9.8744, + "58": 10.14007, + "59": 9.93447, + "60": 9.84864, + "61": 9.98549, + "62": 10.2164, + "63": 9.69034, + "64": 10.0182, + "65": 9.30046, + "66": 9.9355, + "67": 9.63051, + "68": 9.99128, + "69": 9.9852, + "70": 9.92463, + "71": 9.81436, + "72": 9.79481, + "73": 9.68082, + "74": 9.1945, + "75": 9.60407, + "76": 9.28537, + "77": 10.18507, + "78": 9.86718, + "79": 9.52407, + "80": 9.55749, + "81": 9.62863, + "82": 9.81568, + "83": 9.45708, + "84": 9.53654, + "85": 9.73266, + "86": 9.20138, + "87": 9.69524, + "88": 9.85412, + "89": 9.71648, + "90": 9.91047, + "91": 9.45992, + "92": 9.46603, + "93": 9.19321, + "94": 8.94, + "95": 9.60607, + "96": 9.62214, + "97": 9.39796, + "98": 9.76023, + "99": 8.99097, + "100": 9.49505 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 454770688.0, + "2": 454770688.0, + "3": 454770688.0, + "4": 454770688.0, + "5": 454770688.0, + "6": 454770688.0, + "7": 454770688.0, + "8": 454770688.0, + "9": 454770688.0, + "10": 454770688.0, + "11": 454770688.0, + "12": 454770688.0, + "13": 454770688.0, + "14": 454770688.0, + "15": 454770688.0, + "16": 454770688.0, + "17": 454770688.0, + "18": 518880768.0, + "19": 518880768.0, + "20": 518880768.0, + "21": 518880768.0, + "22": 518880768.0, + "23": 518880768.0, + "24": 518880768.0, + "25": 518880768.0, + "26": 518880768.0, + "27": 518880768.0, + "28": 518880768.0, + "29": 518880768.0, + "30": 518880768.0, + "31": 518880768.0, + "32": 518880768.0, + "33": 518880768.0, + "34": 518880768.0, + "35": 518880768.0, + "36": 518880768.0, + "37": 518880768.0, + "38": 518880768.0, + "39": 518880768.0, + "40": 518880768.0, + "41": 518880768.0, + "42": 518880768.0, + "43": 518880768.0, + "44": 518880768.0, + "45": 518880768.0, + "46": 518880768.0, + "47": 518880768.0, + "48": 518880768.0, + "49": 518880768.0, + "50": 518880768.0, + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, + "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, + "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, + "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, + "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, + "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, + "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, + "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, + "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, + "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, + "100": 518880768.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4511150592.0, + "2": 4544705536.0, + "3": 4544705536.0, + "4": 4544705536.0, + "5": 4544705536.0, + "6": 4544705536.0, + "7": 4544705536.0, + "8": 4544705536.0, + "9": 4544705536.0, + "10": 4544705536.0, + "11": 4544705536.0, + "12": 4544705536.0, + "13": 4544705536.0, + "14": 4544705536.0, + "15": 4544705536.0, + "16": 4544705536.0, + "17": 4544705536.0, + "18": 4544705536.0, + "19": 4607767040.0, + "20": 4607767040.0, + "21": 4607767040.0, + "22": 4607767040.0, + "23": 4607767040.0, + "24": 4607767040.0, + "25": 4607767040.0, + "26": 4607767040.0, + "27": 4607767040.0, + "28": 4607767040.0, + "29": 4607767040.0, + "30": 4607767040.0, + "31": 4607767040.0, + "32": 4607767040.0, + "33": 4607767040.0, + "34": 4607767040.0, + "35": 4607767040.0, + "36": 4607767040.0, + "37": 4607767040.0, + "38": 4607767040.0, + "39": 4607767040.0, + "40": 4607767040.0, + "41": 4607767040.0, + "42": 4607767040.0, + "43": 4607767040.0, + "44": 4607767040.0, + "45": 4607767040.0, + "46": 4607767040.0, + "47": 4607767040.0, + "48": 4607767040.0, + "49": 4607767040.0, + "50": 4607767040.0, + "51": 4607767040.0, + "52": 4607767040.0, + "53": 4607767040.0, + "54": 4607767040.0, + "55": 4607767040.0, + "56": 4607767040.0, + "57": 4607767040.0, + "58": 4607767040.0, + "59": 4607767040.0, + "60": 4607767040.0, + "61": 4607767040.0, + "62": 4607767040.0, + "63": 4607767040.0, + "64": 4607767040.0, + "65": 4607767040.0, + "66": 4607767040.0, + "67": 4607767040.0, + "68": 4607767040.0, + "69": 4607767040.0, + "70": 4607767040.0, + "71": 4607767040.0, + "72": 4607767040.0, + "73": 4607767040.0, + "74": 4607767040.0, + "75": 4607767040.0, + "76": 4607767040.0, + "77": 4607767040.0, + "78": 4607767040.0, + "79": 4607767040.0, + "80": 4607767040.0, + "81": 4607767040.0, + "82": 4607767040.0, + "83": 4607767040.0, + "84": 4607767040.0, + "85": 4607767040.0, + "86": 4607767040.0, + "87": 4607767040.0, + "88": 4607767040.0, + "89": 4607767040.0, + "90": 4607767040.0, + "91": 4607767040.0, + "92": 4607767040.0, + "93": 4607767040.0, + "94": 4607767040.0, + "95": 4607767040.0, + "96": 4607767040.0, + "97": 4607767040.0, + "98": 4607767040.0, + "99": 4607767040.0, + "100": 4607767040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 6.67261, + "2": 0.13779, + "3": 0.09189, + "4": 0.08152, + "5": 0.07921, + "6": 0.08183, + "7": 0.07954, + "8": 0.08189, + "9": 0.07832, + "10": 0.08246, + "11": 0.07887, + "12": 0.08149, + "13": 0.07923, + "14": 0.08212, + "15": 0.07877, + "16": 0.08237, + "17": 0.07817, + "18": 0.13039, + "19": 0.08532, + "20": 0.08992, + "21": 0.08507, + "22": 0.08353, + "23": 0.08553, + "24": 0.08796, + "25": 0.08473, + "26": 0.08789, + "27": 0.08546, + "28": 0.08965, + "29": 0.08525, + "30": 0.08688, + "31": 0.0848, + "32": 0.08641, + "33": 0.08391, + "34": 0.08656, + "35": 0.08377, + "36": 0.08671, + "37": 0.08444, + "38": 0.08702, + "39": 0.08436, + "40": 0.08684, + "41": 0.08451, + "42": 0.08735, + "43": 0.08464, + "44": 0.08691, + "45": 0.08466, + "46": 0.08716, + "47": 0.08473, + "48": 0.08709, + "49": 0.08554, + "50": 0.08789, + "51": 0.09019, + "52": 0.08586, + "53": 0.08501, + "54": 0.09024, + "55": 0.08495, + "56": 0.08576, + "57": 0.08467, + "58": 0.08551, + "59": 0.08467, + "60": 0.08553, + "61": 0.0844, + "62": 0.08515, + "63": 0.0844, + "64": 0.08544, + "65": 0.08458, + "66": 0.08526, + "67": 0.08524, + "68": 0.08505, + "69": 0.08429, + "70": 0.08455, + "71": 0.08366, + "72": 0.08448, + "73": 0.08424, + "74": 0.08543, + "75": 0.0846, + "76": 0.08561, + "77": 0.08398, + "78": 0.08524, + "79": 0.08416, + "80": 0.08532, + "81": 0.08438, + "82": 0.08463, + "83": 0.08446, + "84": 0.08488, + "85": 0.0843, + "86": 0.08451, + "87": 0.08396, + "88": 0.08429, + "89": 0.08428, + "90": 0.08516, + "91": 0.08478, + "92": 0.08525, + "93": 0.08465, + "94": 0.08458, + "95": 0.08447, + "96": 0.08467, + "97": 0.08396, + "98": 0.08446, + "99": 0.08448, + "100": 0.08434 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": 1208.0, + "19": 1541.0, + "20": 1173.0, + "21": 1368.0, + "22": "nan", + "23": 1317.0, + "24": 1131.0, + "25": 1119.0, + "26": 1067.0, + "27": 1300.0, + "28": 1549.0, + "29": 1501.0, + "30": 1354.0, + "31": 1044.0, + "32": 1202.0, + "33": 1503.0, + "34": 1276.0, + "35": 1207.0, + "36": 1217.0, + "37": 1430.0, + "38": 1473.0, + "39": 1525.0, + "40": 1413.0, + "41": 1441.0, + "42": 1487.0, + "43": 1151.0, + "44": 1217.0, + "45": 1314.0, + "46": 1265.0, + "47": 1868.0, + "48": 1314.0, + "49": 1105.0, + "50": 1476.0, + "51": 1508.0, + "52": 1400.0, + "53": 1740.0, + "54": 1498.0, + "55": 1551.0, + "56": 1363.0, + "57": 1465.0, + "58": 1610.0, + "59": 1574.0, + "60": 1599.0, + "61": 1727.0, + "62": 1804.0, + "63": 1590.0, + "64": 1813.0, + "65": 1398.0, + "66": 1738.0, + "67": 1536.0, + "68": 1764.0, + "69": 1781.0, + "70": 1926.0, + "71": 1950.0, + "72": 1461.0, + "73": 1985.0, + "74": 1345.0, + "75": 1871.0, + "76": 1732.0, + "77": 2086.0, + "78": 2075.0, + "79": 1992.0, + "80": 2260.0, + "81": 2300.0, + "82": 2290.0, + "83": 1774.0, + "84": 2172.0, + "85": 2216.0, + "86": 2038.0, + "87": 2741.0, + "88": 2079.0, + "89": 2349.0, + "90": 2315.0, + "91": 1875.0, + "92": 2611.0, + "93": 2053.0, + "94": 2220.0, + "95": 2296.0, + "96": 2665.0, + "97": 2516.0, + "98": 2549.0, + "99": 2378.0, + "100": 2257.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json new file mode 100644 index 00000000000..bd904cfc4ab --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.83572, + "2": 10.8327, + "3": 10.83034, + "4": 10.80223, + "5": 10.85556, + "6": 10.86667, + "7": 10.84637, + "8": 10.84251, + "9": 10.85429, + "10": 10.80752, + "11": 10.88921, + "12": 10.87043, + "13": 10.87395, + "14": 10.88615, + "15": 10.79982, + "16": 10.81152, + "17": 10.78946, + "18": 10.82078, + "19": 10.80788, + "20": 10.71283, + "21": 10.68647, + "22": 10.54255, + "23": 10.71926, + "24": 10.58463, + "25": 10.54402, + "26": 10.61055, + "27": 10.61018, + "28": 10.57117, + "29": 10.59062, + "30": 10.35312, + "31": 10.0924, + "32": 10.46991, + "33": 10.45536, + "34": 10.19985, + "35": 10.26269, + "36": 10.21197, + "37": 10.33645, + "38": 10.17551, + "39": 10.39309, + "40": 10.07155, + "41": 10.13767, + "42": 10.19633, + "43": 9.81246, + "44": 9.93615, + "45": 9.81584, + "46": 9.80638, + "47": 10.12803, + "48": 9.82444, + "49": 9.50618, + "50": 9.89067 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1564.0, + "2": 1660.0, + "3": 1617.0, + "4": 1711.0, + "5": 1840.0, + "6": 1754.0, + "7": 1793.0, + "8": 1599.0, + "9": 1894.0, + "10": 1424.0, + "11": 1793.0, + "12": 1727.0, + "13": 1856.0, + "14": 1761.0, + "15": 1915.0, + "16": 1859.0, + "17": 1727.0, + "18": 1585.0, + "19": 1747.0, + "20": 1655.0, + "21": 1783.0, + "22": 1671.0, + "23": 2028.0, + "24": 1698.0, + "25": 1618.0, + "26": 1708.0, + "27": 1804.0, + "28": 1978.0, + "29": 1950.0, + "30": 1911.0, + "31": 1582.0, + "32": 1852.0, + "33": 2049.0, + "34": 1912.0, + "35": 1999.0, + "36": 1972.0, + "37": 2236.0, + "38": 2294.0, + "39": 2209.0, + "40": 2171.0, + "41": 2340.0, + "42": 2153.0, + "43": 2015.0, + "44": 2122.0, + "45": 2123.0, + "46": 2194.0, + "47": 2463.0, + "48": 2382.0, + "49": 2300.0, + "50": 2397.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 952847360.0, + "2": 952847360.0, + "3": 952847360.0, + "4": 952847360.0, + "5": 952847360.0, + "6": 952847360.0, + "7": 952847360.0, + "8": 952847360.0, + "9": 952847360.0, + "10": 952847360.0, + "11": 952847360.0, + "12": 952847360.0, + "13": 952847360.0, + "14": 952847360.0, + "15": 952847360.0, + "16": 952847360.0, + "17": 952847360.0, + "18": 952847360.0, + "19": 952847360.0, + "20": 952847360.0, + "21": 952847360.0, + "22": 952847360.0, + "23": 952847360.0, + "24": 952847360.0, + "25": 952847360.0, + "26": 952847360.0, + "27": 952847360.0, + "28": 952847360.0, + "29": 952847360.0, + "30": 952847360.0, + "31": 952847360.0, + "32": 952847360.0, + "33": 952847360.0, + "34": 952847360.0, + "35": 952847360.0, + "36": 952847360.0, + "37": 952847360.0, + "38": 952847360.0, + "39": 952847360.0, + "40": 952847360.0, + "41": 952847360.0, + "42": 952847360.0, + "43": 952847360.0, + "44": 952847360.0, + "45": 952847360.0, + "46": 952847360.0, + "47": 952847360.0, + "48": 952847360.0, + "49": 952847360.0, + "50": 952847360.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3275808768.0, + "2": 3637371904.0, + "3": 3637371904.0, + "4": 3637371904.0, + "5": 3637371904.0, + "6": 3637371904.0, + "7": 3637371904.0, + "8": 3637371904.0, + "9": 3637371904.0, + "10": 3637371904.0, + "11": 3637371904.0, + "12": 3637371904.0, + "13": 3637371904.0, + "14": 3637371904.0, + "15": 3637371904.0, + "16": 3637371904.0, + "17": 3637371904.0, + "18": 3637371904.0, + "19": 3637371904.0, + "20": 3637371904.0, + "21": 3637371904.0, + "22": 3637371904.0, + "23": 3637371904.0, + "24": 3637371904.0, + "25": 3637371904.0, + "26": 3637371904.0, + "27": 3637371904.0, + "28": 3637371904.0, + "29": 3637371904.0, + "30": 3637371904.0, + "31": 3637371904.0, + "32": 3637371904.0, + "33": 3637371904.0, + "34": 3637371904.0, + "35": 3637371904.0, + "36": 3637371904.0, + "37": 3637371904.0, + "38": 3637371904.0, + "39": 3637371904.0, + "40": 3637371904.0, + "41": 3637371904.0, + "42": 3637371904.0, + "43": 3637371904.0, + "44": 3637371904.0, + "45": 3637371904.0, + "46": 3637371904.0, + "47": 3637371904.0, + "48": 3637371904.0, + "49": 3637371904.0, + "50": 3637371904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6.10738, + "2": 0.17901, + "3": 0.1397, + "4": 0.13942, + "5": 0.14017, + "6": 0.13965, + "7": 0.14063, + "8": 0.13557, + "9": 0.13768, + "10": 0.13663, + "11": 0.1362, + "12": 0.13692, + "13": 0.13568, + "14": 0.1363, + "15": 0.1353, + "16": 0.13498, + "17": 0.13404, + "18": 0.13485, + "19": 0.13514, + "20": 0.13617, + "21": 0.13536, + "22": 0.13562, + "23": 0.13545, + "24": 0.13477, + "25": 0.13399, + "26": 0.13564, + "27": 0.13568, + "28": 0.13537, + "29": 0.13525, + "30": 0.13583, + "31": 0.13497, + "32": 0.13567, + "33": 0.13505, + "34": 0.135, + "35": 0.13548, + "36": 0.13443, + "37": 0.13673, + "38": 0.1352, + "39": 0.13515, + "40": 0.13515, + "41": 0.13635, + "42": 0.13577, + "43": 0.13589, + "44": 0.13513, + "45": 0.1355, + "46": 0.13566, + "47": 0.13559, + "48": 0.13616, + "49": 0.1357, + "50": 0.13591 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..7daadfdadc2 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.83568, + "2": 10.83266, + "3": 10.83151, + "4": 10.80343, + "5": 10.8567, + "6": 10.86778, + "7": 10.84836, + "8": 10.84624, + "9": 10.85924, + "10": 10.81478, + "11": 10.89821, + "12": 10.88433, + "13": 10.88963, + "14": 10.90075, + "15": 10.85098, + "16": 10.86603, + "17": 10.85455, + "18": 10.88507, + "19": 10.8773, + "20": 10.85865, + "21": 10.85654, + "22": 10.79685, + "23": 10.88724, + "24": 10.82649, + "25": 10.81343, + "26": 10.82705, + "27": 10.84612, + "28": 10.84227, + "29": 10.85329, + "30": 10.74969, + "31": 10.63041, + "32": 10.79004, + "33": 10.77234, + "34": 10.65722, + "35": 10.65857, + "36": 10.61583, + "37": 10.67536, + "38": 10.58101, + "39": 10.69083, + "40": 10.50359, + "41": 10.52777, + "42": 10.55371, + "43": 10.28636, + "44": 10.36369, + "45": 10.2738, + "46": 10.24567, + "47": 10.45103, + "48": 10.23707, + "49": 9.99555, + "50": 10.25588, + "51": 10.20129, + "52": 10.10855, + "53": 10.34609, + "54": 10.24857, + "55": 10.18782, + "56": 9.95521, + "57": 9.81221, + "58": 10.10875, + "59": 9.8863, + "60": 9.80901, + "61": 9.94824, + "62": 10.1999, + "63": 9.64431, + "64": 9.9951, + "65": 9.24475, + "66": 9.90917, + "67": 9.59735, + "68": 9.97285, + "69": 9.96332, + "70": 9.91039, + "71": 9.78596, + "72": 9.77263, + "73": 9.6618, + "74": 9.16289, + "75": 9.5812, + "76": 9.26137, + "77": 10.17615, + "78": 9.85644, + "79": 9.50644, + "80": 9.54102, + "81": 9.61313, + "82": 9.80669, + "83": 9.44696, + "84": 9.52782, + "85": 9.72633, + "86": 9.19099, + "87": 9.68736, + "88": 9.85216, + "89": 9.71335, + "90": 9.90316, + "91": 9.46064, + "92": 9.46059, + "93": 9.19418, + "94": 8.93434, + "95": 9.60258, + "96": 9.61852, + "97": 9.39594, + "98": 9.76012, + "99": 8.98668, + "100": 9.49405 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 684610560.0, + "2": 685659136.0, + "3": 685659136.0, + "4": 685659136.0, + "5": 685659136.0, + "6": 685659136.0, + "7": 685659136.0, + "8": 685659136.0, + "9": 685659136.0, + "10": 685659136.0, + "11": 685659136.0, + "12": 685659136.0, + "13": 685659136.0, + "14": 685659136.0, + "15": 685659136.0, + "16": 685659136.0, + "17": 1043027456.0, + "18": 1043027456.0, + "19": 1043027456.0, + "20": 1043027456.0, + "21": 1043027456.0, + "22": 1043027456.0, + "23": 1043027456.0, + "24": 1043027456.0, + "25": 1043027456.0, + "26": 1043027456.0, + "27": 1043027456.0, + "28": 1043027456.0, + "29": 1043027456.0, + "30": 1043027456.0, + "31": 1043027456.0, + "32": 1043027456.0, + "33": 1043027456.0, + "34": 1043027456.0, + "35": 1043027456.0, + "36": 1043027456.0, + "37": 1043027456.0, + "38": 1043027456.0, + "39": 1043027456.0, + "40": 1043027456.0, + "41": 1043027456.0, + "42": 1043027456.0, + "43": 1043027456.0, + "44": 1043027456.0, + "45": 1043027456.0, + "46": 1043027456.0, + "47": 1043027456.0, + "48": 1043027456.0, + "49": 1043027456.0, + "50": 1043027456.0, + "51": 1043027456.0, + "52": 1043027456.0, + "53": 1043027456.0, + "54": 1043027456.0, + "55": 1043027456.0, + "56": 1043027456.0, + "57": 1043027456.0, + "58": 1043027456.0, + "59": 1043027456.0, + "60": 1043027456.0, + "61": 1043027456.0, + "62": 1043027456.0, + "63": 1043027456.0, + "64": 1043027456.0, + "65": 1043027456.0, + "66": 1043027456.0, + "67": 1043027456.0, + "68": 1043027456.0, + "69": 1043027456.0, + "70": 1043027456.0, + "71": 1043027456.0, + "72": 1043027456.0, + "73": 1043027456.0, + "74": 1043027456.0, + "75": 1043027456.0, + "76": 1043027456.0, + "77": 1043027456.0, + "78": 1043027456.0, + "79": 1043027456.0, + "80": 1043027456.0, + "81": 1043027456.0, + "82": 1043027456.0, + "83": 1043027456.0, + "84": 1043027456.0, + "85": 1043027456.0, + "86": 1043027456.0, + "87": 1043027456.0, + "88": 1043027456.0, + "89": 1043027456.0, + "90": 1043027456.0, + "91": 1043027456.0, + "92": 1043027456.0, + "93": 1043027456.0, + "94": 1043027456.0, + "95": 1043027456.0, + "96": 1043027456.0, + "97": 1043027456.0, + "98": 1043027456.0, + "99": 1043027456.0, + "100": 1043027456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3187304960.0, + "2": 3187305472.0, + "3": 3187305472.0, + "4": 3187305472.0, + "5": 3187305472.0, + "6": 3187305472.0, + "7": 3187305472.0, + "8": 3187305472.0, + "9": 3187305472.0, + "10": 3187305472.0, + "11": 3187305472.0, + "12": 3187305472.0, + "13": 3187305472.0, + "14": 3187305472.0, + "15": 3187305472.0, + "16": 3187305472.0, + "17": 3187305472.0, + "18": 3544935936.0, + "19": 3544935936.0, + "20": 3544935936.0, + "21": 3544935936.0, + "22": 3544935936.0, + "23": 3544935936.0, + "24": 3544935936.0, + "25": 3544935936.0, + "26": 3544935936.0, + "27": 3544935936.0, + "28": 3544935936.0, + "29": 3544935936.0, + "30": 3544935936.0, + "31": 3544935936.0, + "32": 3544935936.0, + "33": 3544935936.0, + "34": 3544935936.0, + "35": 3544935936.0, + "36": 3544935936.0, + "37": 3544935936.0, + "38": 3544935936.0, + "39": 3544935936.0, + "40": 3544935936.0, + "41": 3544935936.0, + "42": 3544935936.0, + "43": 3544935936.0, + "44": 3544935936.0, + "45": 3544935936.0, + "46": 3544935936.0, + "47": 3544935936.0, + "48": 3544935936.0, + "49": 3544935936.0, + "50": 3544935936.0, + "51": 3544935936.0, + "52": 3544935936.0, + "53": 3544935936.0, + "54": 3544935936.0, + "55": 3544935936.0, + "56": 3544935936.0, + "57": 3544935936.0, + "58": 3544935936.0, + "59": 3544935936.0, + "60": 3544935936.0, + "61": 3544935936.0, + "62": 3544935936.0, + "63": 3544935936.0, + "64": 3544935936.0, + "65": 3544935936.0, + "66": 3544935936.0, + "67": 3544935936.0, + "68": 3544935936.0, + "69": 3544935936.0, + "70": 3544935936.0, + "71": 3544935936.0, + "72": 3544935936.0, + "73": 3544935936.0, + "74": 3544935936.0, + "75": 3544935936.0, + "76": 3544935936.0, + "77": 3544935936.0, + "78": 3544935936.0, + "79": 3544935936.0, + "80": 3544935936.0, + "81": 3544935936.0, + "82": 3544935936.0, + "83": 3544935936.0, + "84": 3544935936.0, + "85": 3544935936.0, + "86": 3544935936.0, + "87": 3544935936.0, + "88": 3544935936.0, + "89": 3544935936.0, + "90": 3544935936.0, + "91": 3544935936.0, + "92": 3544935936.0, + "93": 3544935936.0, + "94": 3544935936.0, + "95": 3544935936.0, + "96": 3544935936.0, + "97": 3544935936.0, + "98": 3544935936.0, + "99": 3544935936.0, + "100": 3544935936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5.70814, + "2": 0.14593, + "3": 0.11555, + "4": 0.11175, + "5": 0.1118, + "6": 0.11213, + "7": 0.11225, + "8": 0.11222, + "9": 0.11272, + "10": 0.1116, + "11": 0.1115, + "12": 0.11176, + "13": 0.11146, + "14": 0.11223, + "15": 0.11173, + "16": 0.11171, + "17": 0.16961, + "18": 0.1294, + "19": 0.1201, + "20": 0.12081, + "21": 0.11987, + "22": 0.11184, + "23": 0.12343, + "24": 0.12287, + "25": 0.12267, + "26": 0.12216, + "27": 0.12233, + "28": 0.12277, + "29": 0.12268, + "30": 0.12277, + "31": 0.12342, + "32": 0.12274, + "33": 0.12253, + "34": 0.12286, + "35": 0.12315, + "36": 0.12347, + "37": 0.1234, + "38": 0.12348, + "39": 0.12285, + "40": 0.12222, + "41": 0.12362, + "42": 0.12485, + "43": 0.12359, + "44": 0.12299, + "45": 0.12361, + "46": 0.12415, + "47": 0.12313, + "48": 0.12346, + "49": 0.12289, + "50": 0.12312, + "51": 0.13855, + "52": 0.13267, + "53": 0.13126, + "54": 0.13154, + "55": 0.13172, + "56": 0.13114, + "57": 0.13151, + "58": 0.13088, + "59": 0.13381, + "60": 0.13088, + "61": 0.13111, + "62": 0.13298, + "63": 0.13201, + "64": 0.13127, + "65": 0.13143, + "66": 0.13109, + "67": 0.13178, + "68": 0.13132, + "69": 0.13213, + "70": 0.13163, + "71": 0.13172, + "72": 0.13145, + "73": 0.13184, + "74": 0.13219, + "75": 0.13199, + "76": 0.13149, + "77": 0.1316, + "78": 0.13139, + "79": 0.13159, + "80": 0.13172, + "81": 0.13308, + "82": 0.13206, + "83": 0.13148, + "84": 0.13094, + "85": 0.13073, + "86": 0.1311, + "87": 0.13131, + "88": 0.1313, + "89": 0.1322, + "90": 0.13069, + "91": 0.13179, + "92": 0.13016, + "93": 0.13189, + "94": 0.13136, + "95": 0.13186, + "96": 0.13166, + "97": 0.13159, + "98": 0.13156, + "99": 0.13072, + "100": 0.13036 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2249.0, + "18": 2165.0, + "19": 2362.0, + "20": 1953.0, + "21": 1898.0, + "22": "nan", + "23": 2371.0, + "24": 1984.0, + "25": 1818.0, + "26": 1980.0, + "27": 2078.0, + "28": 2467.0, + "29": 2395.0, + "30": 2298.0, + "31": 1682.0, + "32": 2236.0, + "33": 2192.0, + "34": 1800.0, + "35": 2083.0, + "36": 2139.0, + "37": 2498.0, + "38": 2218.0, + "39": 2642.0, + "40": 2287.0, + "41": 2344.0, + "42": 2304.0, + "43": 2098.0, + "44": 2107.0, + "45": 2243.0, + "46": 1960.0, + "47": 2729.0, + "48": 2418.0, + "49": 1910.0, + "50": 2426.0, + "51": 2335.0, + "52": 2407.0, + "53": 2888.0, + "54": 2477.0, + "55": 2440.0, + "56": 2286.0, + "57": 2340.0, + "58": 2652.0, + "59": 2321.0, + "60": 2493.0, + "61": 2812.0, + "62": 2711.0, + "63": 2367.0, + "64": 2802.0, + "65": 2411.0, + "66": 2869.0, + "67": 2577.0, + "68": 2859.0, + "69": 2524.0, + "70": 3119.0, + "71": 2926.0, + "72": 2251.0, + "73": 2929.0, + "74": 2110.0, + "75": 2884.0, + "76": 2992.0, + "77": 3380.0, + "78": 3484.0, + "79": 3533.0, + "80": 3549.0, + "81": 3616.0, + "82": 3347.0, + "83": 3124.0, + "84": 3276.0, + "85": 3721.0, + "86": 3207.0, + "87": 3941.0, + "88": 3250.0, + "89": 3863.0, + "90": 3452.0, + "91": 2630.0, + "92": 3431.0, + "93": 3123.0, + "94": 3671.0, + "95": 3340.0, + "96": 3874.0, + "97": 3519.0, + "98": 3727.0, + "99": 3447.0, + "100": 3338.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json new file mode 100644 index 00000000000..54bb4a034f3 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79432, + "2": 10.80634, + "3": 10.80586, + "4": 10.77237, + "5": 10.84693, + "6": 10.86605, + "7": 10.82692, + "8": 10.81129, + "9": 10.8342, + "10": 10.76876, + "11": 10.89167, + "12": 10.84477, + "13": 10.85758, + "14": 10.88197, + "15": 10.78831, + "16": 10.7793, + "17": 10.75446, + "18": 10.79296, + "19": 10.79494, + "20": 10.67877, + "21": 10.65858, + "22": 10.50081, + "23": 10.71065, + "24": 10.55089, + "25": 10.50321, + "26": 10.58033, + "27": 10.58262, + "28": 10.55556, + "29": 10.56003, + "30": 10.32992, + "31": 10.08344, + "32": 10.44434, + "33": 10.44238, + "34": 10.19765, + "35": 10.25197, + "36": 10.19117, + "37": 10.31772, + "38": 10.1631, + "39": 10.37486, + "40": 10.05284, + "41": 10.1344, + "42": 10.18877, + "43": 9.80641, + "44": 9.92687, + "45": 9.80332, + "46": 9.811, + "47": 10.12605, + "48": 9.82455, + "49": 9.50975, + "50": 9.88831 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1620.0, + "2": 1664.0, + "3": 1748.0, + "4": 1757.0, + "5": 1754.0, + "6": 1841.0, + "7": 1877.0, + "8": 1697.0, + "9": 1886.0, + "10": 1404.0, + "11": 1889.0, + "12": 1689.0, + "13": 1918.0, + "14": 1757.0, + "15": 1913.0, + "16": 1860.0, + "17": 1797.0, + "18": 1655.0, + "19": 1784.0, + "20": 1616.0, + "21": 1887.0, + "22": 1751.0, + "23": 2100.0, + "24": 1717.0, + "25": 1696.0, + "26": 1723.0, + "27": 1819.0, + "28": 1980.0, + "29": 1962.0, + "30": 2046.0, + "31": 1562.0, + "32": 1935.0, + "33": 2182.0, + "34": 1919.0, + "35": 1994.0, + "36": 1947.0, + "37": 2436.0, + "38": 2218.0, + "39": 2319.0, + "40": 2278.0, + "41": 2348.0, + "42": 2258.0, + "43": 1967.0, + "44": 2011.0, + "45": 2215.0, + "46": 2291.0, + "47": 2519.0, + "48": 2517.0, + "49": 2334.0, + "50": 2325.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 779997184.0, + "2": 779997184.0, + "3": 779997184.0, + "4": 779997184.0, + "5": 779997184.0, + "6": 779997184.0, + "7": 779997184.0, + "8": 779997184.0, + "9": 779997184.0, + "10": 779997184.0, + "11": 779997184.0, + "12": 779997184.0, + "13": 779997184.0, + "14": 779997184.0, + "15": 779997184.0, + "16": 779997184.0, + "17": 779997184.0, + "18": 779997184.0, + "19": 779997184.0, + "20": 779997184.0, + "21": 779997184.0, + "22": 779997184.0, + "23": 779997184.0, + "24": 779997184.0, + "25": 779997184.0, + "26": 779997184.0, + "27": 779997184.0, + "28": 779997184.0, + "29": 779997184.0, + "30": 779997184.0, + "31": 779997184.0, + "32": 779997184.0, + "33": 779997184.0, + "34": 779997184.0, + "35": 779997184.0, + "36": 779997184.0, + "37": 779997184.0, + "38": 779997184.0, + "39": 779997184.0, + "40": 779997184.0, + "41": 779997184.0, + "42": 779997184.0, + "43": 779997184.0, + "44": 779997184.0, + "45": 779997184.0, + "46": 779997184.0, + "47": 779997184.0, + "48": 779997184.0, + "49": 779997184.0, + "50": 779997184.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2463815168.0, + "2": 2746575872.0, + "3": 2746575872.0, + "4": 2746575872.0, + "5": 2746575872.0, + "6": 2746575872.0, + "7": 2746575872.0, + "8": 2746575872.0, + "9": 2746575872.0, + "10": 2746575872.0, + "11": 2746575872.0, + "12": 2746575872.0, + "13": 2746575872.0, + "14": 2746575872.0, + "15": 2746575872.0, + "16": 2746575872.0, + "17": 2746575872.0, + "18": 2746575872.0, + "19": 2746575872.0, + "20": 2746575872.0, + "21": 2746575872.0, + "22": 2746575872.0, + "23": 2746575872.0, + "24": 2746575872.0, + "25": 2746575872.0, + "26": 2746575872.0, + "27": 2746575872.0, + "28": 2746575872.0, + "29": 2746575872.0, + "30": 2746575872.0, + "31": 2746575872.0, + "32": 2746575872.0, + "33": 2746575872.0, + "34": 2746575872.0, + "35": 2746575872.0, + "36": 2746575872.0, + "37": 2746575872.0, + "38": 2746575872.0, + "39": 2746575872.0, + "40": 2746575872.0, + "41": 2746575872.0, + "42": 2746575872.0, + "43": 2746575872.0, + "44": 2746575872.0, + "45": 2746575872.0, + "46": 2746575872.0, + "47": 2746575872.0, + "48": 2746575872.0, + "49": 2746575872.0, + "50": 2746575872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.90003, + "2": 0.19955, + "3": 0.16574, + "4": 0.16319, + "5": 0.1633, + "6": 0.16402, + "7": 0.16471, + "8": 0.16355, + "9": 0.1615, + "10": 0.16252, + "11": 0.16238, + "12": 0.16235, + "13": 0.16169, + "14": 0.16245, + "15": 0.16182, + "16": 0.16332, + "17": 0.16281, + "18": 0.16158, + "19": 0.16119, + "20": 0.16285, + "21": 0.16238, + "22": 0.16274, + "23": 0.16156, + "24": 0.16227, + "25": 0.16211, + "26": 0.16153, + "27": 0.1623, + "28": 0.1629, + "29": 0.16355, + "30": 0.16276, + "31": 0.16161, + "32": 0.16187, + "33": 0.16259, + "34": 0.16234, + "35": 0.16349, + "36": 0.16158, + "37": 0.16308, + "38": 0.16385, + "39": 0.16404, + "40": 0.16221, + "41": 0.16276, + "42": 0.16317, + "43": 0.16353, + "44": 0.16159, + "45": 0.16259, + "46": 0.16401, + "47": 0.16514, + "48": 0.16591, + "49": 0.1642, + "50": 0.16529 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..27848326364 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.79449, + "2": 10.80656, + "3": 10.80727, + "4": 10.77389, + "5": 10.84829, + "6": 10.86736, + "7": 10.82922, + "8": 10.81537, + "9": 10.83956, + "10": 10.77652, + "11": 10.90107, + "12": 10.85927, + "13": 10.87396, + "14": 10.89723, + "15": 10.83961, + "16": 10.83508, + "17": 10.82101, + "18": 10.86029, + "19": 10.86558, + "20": 10.82896, + "21": 10.83275, + "22": 10.75286, + "23": 10.88062, + "24": 10.78219, + "25": 10.76607, + "26": 10.79522, + "27": 10.79866, + "28": 10.81697, + "29": 10.82169, + "30": 10.69891, + "31": 10.55698, + "32": 10.75759, + "33": 10.74362, + "34": 10.59976, + "35": 10.61772, + "36": 10.56389, + "37": 10.63614, + "38": 10.53029, + "39": 10.65358, + "40": 10.44072, + "41": 10.49636, + "42": 10.50954, + "43": 10.22362, + "44": 10.30902, + "45": 10.21065, + "46": 10.19943, + "47": 10.41641, + "48": 10.18128, + "49": 9.94311, + "50": 10.21224, + "51": 10.16759, + "52": 10.06895, + "53": 10.30707, + "54": 10.20911, + "55": 10.15688, + "56": 9.91474, + "57": 9.77696, + "58": 10.07417, + "59": 9.86333, + "60": 9.77328, + "61": 9.9292, + "62": 10.17156, + "63": 9.62041, + "64": 9.97113, + "65": 9.21979, + "66": 9.88693, + "67": 9.58363, + "68": 9.94922, + "69": 9.95271, + "70": 9.89312, + "71": 9.77658, + "72": 9.75435, + "73": 9.6497, + "74": 9.1439, + "75": 9.56121, + "76": 9.25111, + "77": 10.17063, + "78": 9.85402, + "79": 9.49965, + "80": 9.53086, + "81": 9.60555, + "82": 9.80179, + "83": 9.43744, + "84": 9.51987, + "85": 9.7196, + "86": 9.18595, + "87": 9.68687, + "88": 9.8443, + "89": 9.70586, + "90": 9.89977, + "91": 9.45029, + "92": 9.45356, + "93": 9.18554, + "94": 8.92968, + "95": 9.59767, + "96": 9.61491, + "97": 9.39084, + "98": 9.75667, + "99": 8.97921, + "100": 9.49001 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 570640384.0, + "2": 570640384.0, + "3": 570640384.0, + "4": 570640384.0, + "5": 570640384.0, + "6": 570640384.0, + "7": 570640384.0, + "8": 570640384.0, + "9": 570640384.0, + "10": 570640384.0, + "11": 570640384.0, + "12": 570640384.0, + "13": 570640384.0, + "14": 570640384.0, + "15": 570640384.0, + "16": 570640384.0, + "17": 852351488.0, + "18": 852351488.0, + "19": 852351488.0, + "20": 852351488.0, + "21": 852351488.0, + "22": 852351488.0, + "23": 852351488.0, + "24": 852351488.0, + "25": 852351488.0, + "26": 852351488.0, + "27": 852351488.0, + "28": 852351488.0, + "29": 852351488.0, + "30": 852351488.0, + "31": 852351488.0, + "32": 852351488.0, + "33": 852351488.0, + "34": 852351488.0, + "35": 852351488.0, + "36": 852351488.0, + "37": 852351488.0, + "38": 852351488.0, + "39": 852351488.0, + "40": 852351488.0, + "41": 852351488.0, + "42": 852351488.0, + "43": 852351488.0, + "44": 852351488.0, + "45": 852351488.0, + "46": 852351488.0, + "47": 852351488.0, + "48": 852351488.0, + "49": 852351488.0, + "50": 852351488.0, + "51": 852351488.0, + "52": 852351488.0, + "53": 852351488.0, + "54": 852351488.0, + "55": 852351488.0, + "56": 852351488.0, + "57": 852351488.0, + "58": 852351488.0, + "59": 852351488.0, + "60": 852351488.0, + "61": 852351488.0, + "62": 852351488.0, + "63": 852351488.0, + "64": 852351488.0, + "65": 852351488.0, + "66": 852351488.0, + "67": 852351488.0, + "68": 852351488.0, + "69": 852351488.0, + "70": 852351488.0, + "71": 852351488.0, + "72": 852351488.0, + "73": 852351488.0, + "74": 852351488.0, + "75": 852351488.0, + "76": 852351488.0, + "77": 852351488.0, + "78": 852351488.0, + "79": 852351488.0, + "80": 852351488.0, + "81": 852351488.0, + "82": 852351488.0, + "83": 852351488.0, + "84": 852351488.0, + "85": 852351488.0, + "86": 852351488.0, + "87": 852351488.0, + "88": 852351488.0, + "89": 852351488.0, + "90": 852351488.0, + "91": 852351488.0, + "92": 852351488.0, + "93": 852351488.0, + "94": 852351488.0, + "95": 852351488.0, + "96": 852351488.0, + "97": 852351488.0, + "98": 852351488.0, + "99": 852351488.0, + "100": 852351488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2394265600.0, + "2": 2394265600.0, + "3": 2394266624.0, + "4": 2394266624.0, + "5": 2394266624.0, + "6": 2394266624.0, + "7": 2394266624.0, + "8": 2394266624.0, + "9": 2394266624.0, + "10": 2394266624.0, + "11": 2394266624.0, + "12": 2394266624.0, + "13": 2394266624.0, + "14": 2394266624.0, + "15": 2394266624.0, + "16": 2394266624.0, + "17": 2394266624.0, + "18": 2675191296.0, + "19": 2675191296.0, + "20": 2675191296.0, + "21": 2675191296.0, + "22": 2675191296.0, + "23": 2675191296.0, + "24": 2675191296.0, + "25": 2675191296.0, + "26": 2675191296.0, + "27": 2675191296.0, + "28": 2675191296.0, + "29": 2675191296.0, + "30": 2675191296.0, + "31": 2675191296.0, + "32": 2675191296.0, + "33": 2675191296.0, + "34": 2675191296.0, + "35": 2675191296.0, + "36": 2675191296.0, + "37": 2675191296.0, + "38": 2675191296.0, + "39": 2675191296.0, + "40": 2675191296.0, + "41": 2675191296.0, + "42": 2675191296.0, + "43": 2675191296.0, + "44": 2675191296.0, + "45": 2675191296.0, + "46": 2675191296.0, + "47": 2675191296.0, + "48": 2675191296.0, + "49": 2675191296.0, + "50": 2675191296.0, + "51": 2675191296.0, + "52": 2675191296.0, + "53": 2675191296.0, + "54": 2675191296.0, + "55": 2675191296.0, + "56": 2675191296.0, + "57": 2675191296.0, + "58": 2675191296.0, + "59": 2675191296.0, + "60": 2675191296.0, + "61": 2675191296.0, + "62": 2675191296.0, + "63": 2675191296.0, + "64": 2675191296.0, + "65": 2675191296.0, + "66": 2675191296.0, + "67": 2675191296.0, + "68": 2675191296.0, + "69": 2675191296.0, + "70": 2675191296.0, + "71": 2675191296.0, + "72": 2675191296.0, + "73": 2675191296.0, + "74": 2675191296.0, + "75": 2675191296.0, + "76": 2675191296.0, + "77": 2675191296.0, + "78": 2675191296.0, + "79": 2675191296.0, + "80": 2675191296.0, + "81": 2675191296.0, + "82": 2675191296.0, + "83": 2675191296.0, + "84": 2675191296.0, + "85": 2675191296.0, + "86": 2675191296.0, + "87": 2675191296.0, + "88": 2675191296.0, + "89": 2675191296.0, + "90": 2675191296.0, + "91": 2675191296.0, + "92": 2675191296.0, + "93": 2675191296.0, + "94": 2675191296.0, + "95": 2675191296.0, + "96": 2675191296.0, + "97": 2675191296.0, + "98": 2675191296.0, + "99": 2675191296.0, + "100": 2675191296.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 7.9608, + "2": 0.18858, + "3": 0.32612, + "4": 0.1506, + "5": 0.14992, + "6": 0.15003, + "7": 0.14956, + "8": 0.15016, + "9": 0.14963, + "10": 0.15221, + "11": 0.1508, + "12": 0.15066, + "13": 0.15018, + "14": 0.15083, + "15": 0.15033, + "16": 0.14989, + "17": 0.20146, + "18": 0.21955, + "19": 0.15566, + "20": 0.15474, + "21": 0.15581, + "22": 0.15525, + "23": 0.15523, + "24": 0.15517, + "25": 0.1553, + "26": 0.16077, + "27": 0.15596, + "28": 0.15597, + "29": 0.15516, + "30": 0.15575, + "31": 0.1558, + "32": 0.15549, + "33": 0.15516, + "34": 0.15477, + "35": 0.15612, + "36": 0.15497, + "37": 0.15537, + "38": 0.15489, + "39": 0.15572, + "40": 0.15493, + "41": 0.15597, + "42": 0.15511, + "43": 0.15627, + "44": 0.15518, + "45": 0.15536, + "46": 0.15569, + "47": 0.15491, + "48": 0.15577, + "49": 0.15552, + "50": 0.15559, + "51": 0.16391, + "52": 0.15616, + "53": 0.15726, + "54": 0.15791, + "55": 0.15557, + "56": 0.15573, + "57": 0.15632, + "58": 0.15547, + "59": 0.15586, + "60": 0.15577, + "61": 0.1549, + "62": 0.15476, + "63": 0.15572, + "64": 0.15478, + "65": 0.15585, + "66": 0.15551, + "67": 0.15545, + "68": 0.15506, + "69": 0.15585, + "70": 0.15461, + "71": 0.15433, + "72": 0.15487, + "73": 0.15474, + "74": 0.15519, + "75": 0.15523, + "76": 0.15503, + "77": 0.15509, + "78": 0.15494, + "79": 0.1546, + "80": 0.15504, + "81": 0.1549, + "82": 0.15457, + "83": 0.15396, + "84": 0.1556, + "85": 0.15751, + "86": 0.15783, + "87": 0.1567, + "88": 0.15639, + "89": 0.15681, + "90": 0.15692, + "91": 0.15773, + "92": 0.15829, + "93": 0.15928, + "94": 0.15804, + "95": 0.15712, + "96": 0.15726, + "97": 0.1578, + "98": 0.15735, + "99": 0.15822, + "100": 0.15812 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": 2437.0, + "18": 2405.0, + "19": 2950.0, + "20": 1827.0, + "21": 2154.0, + "22": 2731.0, + "23": 2609.0, + "24": 2290.0, + "25": 2325.0, + "26": 2079.0, + "27": 2138.0, + "28": 2702.0, + "29": 2576.0, + "30": 2528.0, + "31": 1895.0, + "32": 2628.0, + "33": 2325.0, + "34": 1928.0, + "35": 2061.0, + "36": 2153.0, + "37": 2600.0, + "38": 2350.0, + "39": 2997.0, + "40": 2053.0, + "41": 3352.0, + "42": 2497.0, + "43": 2867.0, + "44": 2109.0, + "45": 2490.0, + "46": 2279.0, + "47": 3051.0, + "48": 2527.0, + "49": 1973.0, + "50": 2887.0, + "51": 2310.0, + "52": 2526.0, + "53": 3705.0, + "54": 2888.0, + "55": 2440.0, + "56": 2496.0, + "57": 2338.0, + "58": 3283.0, + "59": 2849.0, + "60": 2893.0, + "61": 2956.0, + "62": 3134.0, + "63": 3275.0, + "64": 3176.0, + "65": 2318.0, + "66": 3857.0, + "67": 2606.0, + "68": 3313.0, + "69": 2826.0, + "70": 3665.0, + "71": 3011.0, + "72": 2693.0, + "73": 3357.0, + "74": 2271.0, + "75": 2955.0, + "76": 3617.0, + "77": 3936.0, + "78": 3951.0, + "79": 4065.0, + "80": 3665.0, + "81": 5191.0, + "82": 3511.0, + "83": 3263.0, + "84": 3876.0, + "85": 4048.0, + "86": 3414.0, + "87": 3980.0, + "88": 3617.0, + "89": 4400.0, + "90": 3695.0, + "91": 2857.0, + "92": 4432.0, + "93": 3494.0, + "94": 4438.0, + "95": 4076.0, + "96": 3948.0, + "97": 4242.0, + "98": 4943.0, + "99": 3861.0, + "100": 3631.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json new file mode 100644 index 00000000000..2234bf512f2 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465367040.0, + "4": 1465367040.0, + "5": 1465367040.0, + "6": 1465367040.0, + "7": 1465367040.0, + "8": 1465367040.0, + "9": 1465367040.0, + "10": 1465367040.0, + "11": 1465367040.0, + "12": 1465367040.0, + "13": 1465367040.0, + "14": 1465367040.0, + "15": 1465367040.0, + "16": 1465367040.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3.97311, + "2": 0.31307, + "3": 0.286, + "4": 0.28812, + "5": 0.28671, + "6": 0.2848, + "7": 0.28491, + "8": 0.28395, + "9": 0.28376, + "10": 0.28363, + "11": 0.28465, + "12": 0.28384, + "13": 0.28553, + "14": 0.28458, + "15": 0.2847, + "16": 0.46551, + "17": 0.28395, + "18": 0.29675, + "19": 0.29649, + "20": 0.29653, + "21": 0.29565, + "22": 0.28068, + "23": 0.29473, + "24": 0.29499, + "25": 0.2952, + "26": 0.29503, + "27": 0.29442, + "28": 0.29486, + "29": 0.29498, + "30": 0.29465, + "31": 0.29412, + "32": 0.29472, + "33": 0.29518, + "34": 0.29455, + "35": 0.29419, + "36": 0.29438, + "37": 0.29493, + "38": 0.29476, + "39": 0.29549, + "40": 0.2951, + "41": 0.29544, + "42": 0.29541, + "43": 0.29476, + "44": 0.29509, + "45": 0.29517, + "46": 0.29599, + "47": 0.29503, + "48": 0.29432, + "49": 0.29437, + "50": 0.29462, + "51": 0.29586, + "52": 0.29356, + "53": 0.29522, + "54": 0.29503, + "55": 0.29499, + "56": 0.29442, + "57": 0.29497, + "58": 0.29557, + "59": 0.29547, + "60": 0.29625, + "61": 0.29386, + "62": 0.31465, + "63": 0.29761, + "64": 0.29393, + "65": 0.2948, + "66": 0.29437, + "67": 0.29373, + "68": 0.29276, + "69": 0.2938, + "70": 0.2933, + "71": 0.29428, + "72": 0.2943, + "73": 0.29482, + "74": 0.29327, + "75": 0.29367, + "76": 0.29331, + "77": 0.2935, + "78": 0.29375, + "79": 0.29562, + "80": 0.29421, + "81": 0.29353, + "82": 0.29391, + "83": 0.29375, + "84": 0.29415, + "85": 0.29357, + "86": 0.29355, + "87": 0.29272, + "88": 0.29262, + "89": 0.29421, + "90": 0.29259, + "91": 0.29437, + "92": 0.29309, + "93": 0.29366, + "94": 0.29321, + "95": 0.29329, + "96": 0.29263, + "97": 0.2933, + "98": 0.29315, + "99": 0.29434, + "100": 0.29385 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json new file mode 100644 index 00000000000..04250cc4ade --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_lts.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.8595, + "2": 10.8576, + "3": 10.85984, + "4": 10.85001, + "5": 10.88364, + "6": 10.88818, + "7": 10.86353, + "8": 10.87179, + "9": 10.87022, + "10": 10.845, + "11": 10.88619, + "12": 10.88637, + "13": 10.89401, + "14": 10.9025, + "15": 10.88061, + "16": 10.88557, + "17": 10.86363, + "18": 10.88903, + "19": 10.87964, + "20": 10.87344, + "21": 10.8919, + "22": 10.83039, + "23": 10.89392, + "24": 10.8602, + "25": 10.83015, + "26": 10.82911, + "27": 10.8537, + "28": 10.84349, + "29": 10.85387, + "30": 10.77206, + "31": 10.672, + "32": 10.78864, + "33": 10.77618, + "34": 10.67579, + "35": 10.67298, + "36": 10.63381, + "37": 10.69227, + "38": 10.60773, + "39": 10.70463, + "40": 10.5185, + "41": 10.5453, + "42": 10.56937, + "43": 10.32495, + "44": 10.3911, + "45": 10.28431, + "46": 10.2732, + "47": 10.48175, + "48": 10.25374, + "49": 10.01592, + "50": 10.27755, + "51": 10.21727, + "52": 10.1271, + "53": 10.36018, + "54": 10.25981, + "55": 10.20104, + "56": 9.98213, + "57": 9.84717, + "58": 10.12257, + "59": 9.90914, + "60": 9.83288, + "61": 9.9713, + "62": 10.22005, + "63": 9.67481, + "64": 10.01706, + "65": 9.27085, + "66": 9.93979, + "67": 9.62899, + "68": 9.98681, + "69": 9.9839, + "70": 9.92559, + "71": 9.81011, + "72": 9.79196, + "73": 9.68163, + "74": 9.17945, + "75": 9.61324, + "76": 9.28951, + "77": 10.19435, + "78": 9.8755, + "79": 9.5297, + "80": 9.56593, + "81": 9.63478, + "82": 9.82295, + "83": 9.47164, + "84": 9.54623, + "85": 9.74358, + "86": 9.20093, + "87": 9.70179, + "88": 9.86553, + "89": 9.73045, + "90": 9.92108, + "91": 9.48732, + "92": 9.47637, + "93": 9.21283, + "94": 8.94903, + "95": 9.6165, + "96": 9.63374, + "97": 9.41244, + "98": 9.7751, + "99": 9.00191, + "100": 9.50967 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 284527616.0, + "2": 284527616.0, + "3": 284527616.0, + "4": 284527616.0, + "5": 284527616.0, + "6": 284527616.0, + "7": 284527616.0, + "8": 284527616.0, + "9": 284527616.0, + "10": 284527616.0, + "11": 284527616.0, + "12": 284527616.0, + "13": 284527616.0, + "14": 284527616.0, + "15": 284527616.0, + "16": 416513536.0, + "17": 416513536.0, + "18": 416513536.0, + "19": 416513536.0, + "20": 416513536.0, + "21": 416513536.0, + "22": 416513536.0, + "23": 416513536.0, + "24": 416513536.0, + "25": 416513536.0, + "26": 416513536.0, + "27": 416513536.0, + "28": 416513536.0, + "29": 416513536.0, + "30": 416513536.0, + "31": 416513536.0, + "32": 416513536.0, + "33": 416513536.0, + "34": 416513536.0, + "35": 416513536.0, + "36": 416513536.0, + "37": 416513536.0, + "38": 416513536.0, + "39": 416513536.0, + "40": 416513536.0, + "41": 416513536.0, + "42": 416513536.0, + "43": 416513536.0, + "44": 416513536.0, + "45": 416513536.0, + "46": 416513536.0, + "47": 416513536.0, + "48": 416513536.0, + "49": 416513536.0, + "50": 416513536.0, + "51": 416513536.0, + "52": 416513536.0, + "53": 416513536.0, + "54": 416513536.0, + "55": 416513536.0, + "56": 416513536.0, + "57": 416513536.0, + "58": 416513536.0, + "59": 416513536.0, + "60": 416513536.0, + "61": 416513536.0, + "62": 416513536.0, + "63": 416513536.0, + "64": 416513536.0, + "65": 416513536.0, + "66": 416513536.0, + "67": 416513536.0, + "68": 416513536.0, + "69": 416513536.0, + "70": 416513536.0, + "71": 416513536.0, + "72": 416513536.0, + "73": 416513536.0, + "74": 416513536.0, + "75": 416513536.0, + "76": 416513536.0, + "77": 416513536.0, + "78": 416513536.0, + "79": 416513536.0, + "80": 416513536.0, + "81": 416513536.0, + "82": 416513536.0, + "83": 416513536.0, + "84": 416513536.0, + "85": 416513536.0, + "86": 416513536.0, + "87": 416513536.0, + "88": 416513536.0, + "89": 416513536.0, + "90": 416513536.0, + "91": 416513536.0, + "92": 416513536.0, + "93": 416513536.0, + "94": 416513536.0, + "95": 416513536.0, + "96": 416513536.0, + "97": 416513536.0, + "98": 416513536.0, + "99": 416513536.0, + "100": 416513536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1465367040.0, + "2": 1465367040.0, + "3": 1465368576.0, + "4": 1465368576.0, + "5": 1465368576.0, + "6": 1465368576.0, + "7": 1465368576.0, + "8": 1465368576.0, + "9": 1465368576.0, + "10": 1465368576.0, + "11": 1465368576.0, + "12": 1465368576.0, + "13": 1465368576.0, + "14": 1465368576.0, + "15": 1465368576.0, + "16": 1465368576.0, + "17": 1597485568.0, + "18": 1597485568.0, + "19": 1597485568.0, + "20": 1597485568.0, + "21": 1597485568.0, + "22": 1597485568.0, + "23": 1597485568.0, + "24": 1597485568.0, + "25": 1597485568.0, + "26": 1597485568.0, + "27": 1597485568.0, + "28": 1597485568.0, + "29": 1597485568.0, + "30": 1597485568.0, + "31": 1597485568.0, + "32": 1597485568.0, + "33": 1597485568.0, + "34": 1597485568.0, + "35": 1597485568.0, + "36": 1597485568.0, + "37": 1597485568.0, + "38": 1597485568.0, + "39": 1597485568.0, + "40": 1597485568.0, + "41": 1597485568.0, + "42": 1597485568.0, + "43": 1597485568.0, + "44": 1597485568.0, + "45": 1597485568.0, + "46": 1597485568.0, + "47": 1597485568.0, + "48": 1597485568.0, + "49": 1597485568.0, + "50": 1597485568.0, + "51": 1597485568.0, + "52": 1597485568.0, + "53": 1597485568.0, + "54": 1597485568.0, + "55": 1597485568.0, + "56": 1597485568.0, + "57": 1597485568.0, + "58": 1597485568.0, + "59": 1597485568.0, + "60": 1597485568.0, + "61": 1597485568.0, + "62": 1597485568.0, + "63": 1597485568.0, + "64": 1597485568.0, + "65": 1597485568.0, + "66": 1597485568.0, + "67": 1597485568.0, + "68": 1597485568.0, + "69": 1597485568.0, + "70": 1597485568.0, + "71": 1597485568.0, + "72": 1597485568.0, + "73": 1597485568.0, + "74": 1597485568.0, + "75": 1597485568.0, + "76": 1597485568.0, + "77": 1597485568.0, + "78": 1597485568.0, + "79": 1597485568.0, + "80": 1597485568.0, + "81": 1597485568.0, + "82": 1597485568.0, + "83": 1597485568.0, + "84": 1597485568.0, + "85": 1597485568.0, + "86": 1597485568.0, + "87": 1597485568.0, + "88": 1597485568.0, + "89": 1597485568.0, + "90": 1597485568.0, + "91": 1597485568.0, + "92": 1597485568.0, + "93": 1597485568.0, + "94": 1597485568.0, + "95": 1597485568.0, + "96": 1597485568.0, + "97": 1597485568.0, + "98": 1597485568.0, + "99": 1597485568.0, + "100": 1597485568.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 4.45088, + "2": 0.31109, + "3": 0.27737, + "4": 0.27658, + "5": 0.27603, + "6": 0.27737, + "7": 0.27608, + "8": 0.27455, + "9": 0.2747, + "10": 0.27491, + "11": 0.27532, + "12": 0.27387, + "13": 0.27333, + "14": 0.2725, + "15": 0.27351, + "16": 0.39563, + "17": 0.28187, + "18": 0.2917, + "19": 0.2907, + "20": 0.29386, + "21": 0.29242, + "22": 0.2778, + "23": 0.29138, + "24": 0.29075, + "25": 0.2915, + "26": 0.29173, + "27": 0.29148, + "28": 0.29145, + "29": 0.29459, + "30": 0.29168, + "31": 0.2922, + "32": 0.29151, + "33": 0.2908, + "34": 0.292, + "35": 0.29122, + "36": 0.29142, + "37": 0.29107, + "38": 0.29211, + "39": 0.29283, + "40": 0.29137, + "41": 0.29251, + "42": 0.29277, + "43": 0.29555, + "44": 0.29375, + "45": 0.29691, + "46": 0.29643, + "47": 0.28907, + "48": 0.28912, + "49": 0.28981, + "50": 0.2881, + "51": 0.29774, + "52": 0.29251, + "53": 0.2899, + "54": 0.29254, + "55": 0.29212, + "56": 0.29296, + "57": 0.29318, + "58": 0.2934, + "59": 0.29268, + "60": 0.29296, + "61": 0.29363, + "62": 0.2935, + "63": 0.29415, + "64": 0.29373, + "65": 0.2936, + "66": 0.29366, + "67": 0.29356, + "68": 0.29364, + "69": 0.29319, + "70": 0.29366, + "71": 0.293, + "72": 0.29339, + "73": 0.29297, + "74": 0.2936, + "75": 0.29303, + "76": 0.29338, + "77": 0.29241, + "78": 0.29316, + "79": 0.29318, + "80": 0.29243, + "81": 0.29301, + "82": 0.29271, + "83": 0.29312, + "84": 0.29263, + "85": 0.29313, + "86": 0.29384, + "87": 0.294, + "88": 0.29293, + "89": 0.29341, + "90": 0.29278, + "91": 0.2925, + "92": 0.29328, + "93": 0.29369, + "94": 0.29244, + "95": 0.29207, + "96": 0.29333, + "97": 0.29327, + "98": 0.2931, + "99": 0.29328, + "100": 0.29381 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": 2308.0, + "17": "nan", + "18": 2336.0, + "19": 2843.0, + "20": 1690.0, + "21": 2021.0, + "22": "nan", + "23": 2506.0, + "24": 2095.0, + "25": 2035.0, + "26": 1875.0, + "27": 2000.0, + "28": 2435.0, + "29": 2512.0, + "30": 2279.0, + "31": 1598.0, + "32": 2494.0, + "33": 2128.0, + "34": 1682.0, + "35": 1868.0, + "36": 1970.0, + "37": 2549.0, + "38": 2069.0, + "39": 2950.0, + "40": 1905.0, + "41": 3069.0, + "42": 2521.0, + "43": 2635.0, + "44": 1835.0, + "45": 2455.0, + "46": 2204.0, + "47": 2799.0, + "48": 2423.0, + "49": 1759.0, + "50": 2677.0, + "51": 2195.0, + "52": 2330.0, + "53": 3549.0, + "54": 2650.0, + "55": 2247.0, + "56": 2422.0, + "57": 2195.0, + "58": 3241.0, + "59": 2626.0, + "60": 2775.0, + "61": 2747.0, + "62": 2926.0, + "63": 2898.0, + "64": 3090.0, + "65": 2245.0, + "66": 3827.0, + "67": 2655.0, + "68": 3117.0, + "69": 2656.0, + "70": 3659.0, + "71": 2819.0, + "72": 2710.0, + "73": 3355.0, + "74": 2210.0, + "75": 2927.0, + "76": 3577.0, + "77": 3727.0, + "78": 3855.0, + "79": 4237.0, + "80": 3462.0, + "81": 5157.0, + "82": 3426.0, + "83": 3234.0, + "84": 3878.0, + "85": 3734.0, + "86": 3184.0, + "87": 4090.0, + "88": 3594.0, + "89": 4234.0, + "90": 3744.0, + "91": 2967.0, + "92": 4509.0, + "93": 3649.0, + "94": 4486.0, + "95": 4215.0, + "96": 3851.0, + "97": 4098.0, + "98": 5029.0, + "99": 3975.0, + "100": 3445.0 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json new file mode 100644 index 00000000000..608cb45d7c1 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81189, + "2": 10.82837, + "3": 10.81793, + "4": 10.80617, + "5": 10.84927, + "6": 10.86922, + "7": 10.84362, + "8": 10.83419, + "9": 10.84988, + "10": 10.78408, + "11": 10.88038, + "12": 10.85362, + "13": 10.86619, + "14": 10.88223, + "15": 10.82311, + "16": 10.81052, + "17": 10.78202, + "18": 10.81541, + "19": 10.81756, + "20": 10.74584, + "21": 10.72627, + "22": 10.60022, + "23": 10.74769, + "24": 10.6365, + "25": 10.59453, + "26": 10.64799, + "27": 10.66378, + "28": 10.64562, + "29": 10.65242, + "30": 10.44192, + "31": 10.21776, + "32": 10.52786, + "33": 10.52222, + "34": 10.30567, + "35": 10.35142, + "36": 10.32502, + "37": 10.42585, + "38": 10.28522, + "39": 10.47826, + "40": 10.19414, + "41": 10.22953, + "42": 10.29398, + "43": 9.95703, + "44": 10.06609, + "45": 9.95939, + "46": 9.92711, + "47": 10.22774, + "48": 9.96376, + "49": 9.65724, + "50": 10.01644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31381.0, + "2": 33910.0, + "3": 33570.0, + "4": 31384.0, + "5": 35903.0, + "6": 37599.0, + "7": 35525.0, + "8": 31635.0, + "9": 34509.0, + "10": 30410.0, + "11": 37757.0, + "12": 35967.0, + "13": 36886.0, + "14": 38227.0, + "15": 35788.0, + "16": 36247.0, + "17": 35378.0, + "18": 35284.0, + "19": 36213.0, + "20": 33124.0, + "21": 33424.0, + "22": 31395.0, + "23": 38414.0, + "24": 32277.0, + "25": 31634.0, + "26": 35264.0, + "27": 35935.0, + "28": 37638.0, + "29": 38145.0, + "30": 33261.0, + "31": 30972.0, + "32": 36646.0, + "33": 38318.0, + "34": 33243.0, + "35": 34813.0, + "36": 35446.0, + "37": 38278.0, + "38": 35590.0, + "39": 38681.0, + "40": 36027.0, + "41": 36509.0, + "42": 37301.0, + "43": 34597.0, + "44": 34024.0, + "45": 35938.0, + "46": 37229.0, + "47": 40862.0, + "48": 35537.0, + "49": 35095.0, + "50": 39711.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 880441344.0, + "2": 880437760.0, + "3": 880442368.0, + "4": 880448512.0, + "5": 880440832.0, + "6": 880441344.0, + "7": 880445440.0, + "8": 880442880.0, + "9": 880440320.0, + "10": 880443392.0, + "11": 880442368.0, + "12": 880439808.0, + "13": 880439808.0, + "14": 880443904.0, + "15": 880443904.0, + "16": 880441856.0, + "17": 880448512.0, + "18": 880437760.0, + "19": 880440320.0, + "20": 880442880.0, + "21": 880445440.0, + "22": 880450048.0, + "23": 880449024.0, + "24": 880440320.0, + "25": 880440320.0, + "26": 880445952.0, + "27": 880441344.0, + "28": 880439808.0, + "29": 880443392.0, + "30": 880441344.0, + "31": 880454656.0, + "32": 880444928.0, + "33": 880440320.0, + "34": 880440832.0, + "35": 880446976.0, + "36": 880441344.0, + "37": 880442880.0, + "38": 880439808.0, + "39": 880441856.0, + "40": 880442880.0, + "41": 880445952.0, + "42": 880441344.0, + "43": 880444416.0, + "44": 880439808.0, + "45": 880440832.0, + "46": 880447488.0, + "47": 880440320.0, + "48": 880441856.0, + "49": 880444416.0, + "50": 880442368.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 2909851648.0, + "2": 3151821824.0, + "3": 3152806912.0, + "4": 3156619264.0, + "5": 3156619264.0, + "6": 3156619264.0, + "7": 3156619264.0, + "8": 3156619264.0, + "9": 3156619264.0, + "10": 3156619264.0, + "11": 3156619264.0, + "12": 3156619264.0, + "13": 3156619264.0, + "14": 3156619264.0, + "15": 3156619264.0, + "16": 3156619264.0, + "17": 3156619264.0, + "18": 3156619264.0, + "19": 3156619264.0, + "20": 3156619264.0, + "21": 3156619264.0, + "22": 3156619264.0, + "23": 3156619264.0, + "24": 3156619264.0, + "25": 3156619264.0, + "26": 3156619264.0, + "27": 3156619264.0, + "28": 3156619264.0, + "29": 3156619264.0, + "30": 3156619264.0, + "31": 3167942656.0, + "32": 3167942656.0, + "33": 3167942656.0, + "34": 3167942656.0, + "35": 3167942656.0, + "36": 3167942656.0, + "37": 3167942656.0, + "38": 3167942656.0, + "39": 3167942656.0, + "40": 3167942656.0, + "41": 3167942656.0, + "42": 3167942656.0, + "43": 3167942656.0, + "44": 3167942656.0, + "45": 3167942656.0, + "46": 3167942656.0, + "47": 3167942656.0, + "48": 3167942656.0, + "49": 3167942656.0, + "50": 3167942656.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.73075, + "2": 0.34394, + "3": 0.30421, + "4": 0.32137, + "5": 0.31439, + "6": 0.295, + "7": 0.29242, + "8": 0.29198, + "9": 0.2927, + "10": 0.2929, + "11": 0.29321, + "12": 0.2902, + "13": 0.29049, + "14": 0.28993, + "15": 0.29425, + "16": 0.29356, + "17": 0.29337, + "18": 0.29277, + "19": 0.29187, + "20": 0.29594, + "21": 0.2932, + "22": 0.29443, + "23": 0.2925, + "24": 0.29533, + "25": 0.29357, + "26": 0.29418, + "27": 0.29544, + "28": 0.28926, + "29": 0.29139, + "30": 0.28991, + "31": 0.2906, + "32": 0.28924, + "33": 0.29092, + "34": 0.28921, + "35": 0.29003, + "36": 0.29075, + "37": 0.28899, + "38": 0.2894, + "39": 0.28854, + "40": 0.289, + "41": 0.28862, + "42": 0.28929, + "43": 0.29092, + "44": 0.28949, + "45": 0.28936, + "46": 0.2894, + "47": 0.28929, + "48": 0.2909, + "49": 0.29399, + "50": 0.29037 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json new file mode 100644 index 00000000000..6c2b5773fc5 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.78519, + "2": 10.81141, + "3": 10.79553, + "4": 10.78363, + "5": 10.83004, + "6": 10.8417, + "7": 10.81276, + "8": 10.81576, + "9": 10.82408, + "10": 10.75834, + "11": 10.86449, + "12": 10.83694, + "13": 10.84627, + "14": 10.85637, + "15": 10.79099, + "16": 10.78206, + "17": 10.73911, + "18": 10.77969, + "19": 10.77703, + "20": 10.69044, + "21": 10.66086, + "22": 10.51537, + "23": 10.68288, + "24": 10.54469, + "25": 10.49929, + "26": 10.5582, + "27": 10.57541, + "28": 10.54334, + "29": 10.55412, + "30": 10.31857, + "31": 10.06911, + "32": 10.42007, + "33": 10.42435, + "34": 10.16897, + "35": 10.22944, + "36": 10.19358, + "37": 10.29976, + "38": 10.1465, + "39": 10.37282, + "40": 10.04287, + "41": 10.09714, + "42": 10.17339, + "43": 9.79317, + "44": 9.91871, + "45": 9.79389, + "46": 9.77586, + "47": 10.10721, + "48": 9.81704, + "49": 9.48887, + "50": 9.87988 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 26764.0, + "2": 29274.0, + "3": 28827.0, + "4": 27434.0, + "5": 31796.0, + "6": 32589.0, + "7": 31434.0, + "8": 27373.0, + "9": 30359.0, + "10": 25775.0, + "11": 33420.0, + "12": 30835.0, + "13": 31910.0, + "14": 33275.0, + "15": 30931.0, + "16": 31713.0, + "17": 31263.0, + "18": 30574.0, + "19": 31200.0, + "20": 28330.0, + "21": 28769.0, + "22": 27643.0, + "23": 34373.0, + "24": 28585.0, + "25": 27693.0, + "26": 30668.0, + "27": 31536.0, + "28": 33620.0, + "29": 34457.0, + "30": 29810.0, + "31": 27999.0, + "32": 33871.0, + "33": 35836.0, + "34": 30770.0, + "35": 32783.0, + "36": 33452.0, + "37": 36437.0, + "38": 34114.0, + "39": 37433.0, + "40": 35063.0, + "41": 34239.0, + "42": 36041.0, + "43": 33976.0, + "44": 33303.0, + "45": 35148.0, + "46": 36112.0, + "47": 38996.0, + "48": 34779.0, + "49": 34913.0, + "50": 38547.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1364012544.0, + "2": 1364020736.0, + "3": 1364009472.0, + "4": 1364020224.0, + "5": 1364007424.0, + "6": 1364005888.0, + "7": 1364011520.0, + "8": 1364006912.0, + "9": 1364019712.0, + "10": 1364001280.0, + "11": 1364009984.0, + "12": 1364022784.0, + "13": 1364008960.0, + "14": 1364003840.0, + "15": 1364007936.0, + "16": 1363994112.0, + "17": 1363993088.0, + "18": 1364017152.0, + "19": 1364009984.0, + "20": 1363999744.0, + "21": 1364006400.0, + "22": 1363983360.0, + "23": 1363997696.0, + "24": 1364001792.0, + "25": 1363992576.0, + "26": 1364003328.0, + "27": 1363998720.0, + "28": 1363982336.0, + "29": 1363995136.0, + "30": 1363979776.0, + "31": 1363991552.0, + "32": 1363983872.0, + "33": 1363983872.0, + "34": 1363997184.0, + "35": 1363991040.0, + "36": 1363984896.0, + "37": 1363998720.0, + "38": 1363991040.0, + "39": 1363976704.0, + "40": 1363971584.0, + "41": 1363983872.0, + "42": 1363977728.0, + "43": 1363967488.0, + "44": 1363974656.0, + "45": 1363974144.0, + "46": 1363965952.0, + "47": 1363952640.0, + "48": 1363951616.0, + "49": 1363960320.0, + "50": 1363956736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3404911104.0, + "2": 3972516352.0, + "3": 3976973312.0, + "4": 3976973312.0, + "5": 3976973312.0, + "6": 3976973312.0, + "7": 3976973312.0, + "8": 3976973312.0, + "9": 3976973312.0, + "10": 3976973312.0, + "11": 3976973312.0, + "12": 3976975872.0, + "13": 3976975872.0, + "14": 3976975872.0, + "15": 3976975872.0, + "16": 3976975872.0, + "17": 3976975872.0, + "18": 3976975872.0, + "19": 3976975872.0, + "20": 3976975872.0, + "21": 3976975872.0, + "22": 3976975872.0, + "23": 3976975872.0, + "24": 3976975872.0, + "25": 3976975872.0, + "26": 3976975872.0, + "27": 3976975872.0, + "28": 3976975872.0, + "29": 3976975872.0, + "30": 3976975872.0, + "31": 3976975872.0, + "32": 3976975872.0, + "33": 3976975872.0, + "34": 3976975872.0, + "35": 3976975872.0, + "36": 3976975872.0, + "37": 3976975872.0, + "38": 3976975872.0, + "39": 3976975872.0, + "40": 3976975872.0, + "41": 3976975872.0, + "42": 3976975872.0, + "43": 3976975872.0, + "44": 3976975872.0, + "45": 3976975872.0, + "46": 3976975872.0, + "47": 3976975872.0, + "48": 3976975872.0, + "49": 3976975872.0, + "50": 3976975872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.62877, + "2": 0.38762, + "3": 0.31182, + "4": 0.30579, + "5": 0.30879, + "6": 0.29821, + "7": 0.29869, + "8": 0.297, + "9": 0.29771, + "10": 0.30139, + "11": 0.29993, + "12": 0.29972, + "13": 0.30066, + "14": 0.29957, + "15": 0.30327, + "16": 0.30397, + "17": 0.29562, + "18": 0.29598, + "19": 0.29565, + "20": 0.29598, + "21": 0.29613, + "22": 0.29515, + "23": 0.29632, + "24": 0.29672, + "25": 0.29668, + "26": 0.29762, + "27": 0.31026, + "28": 0.33283, + "29": 0.29963, + "30": 0.32191, + "31": 0.32754, + "32": 0.30244, + "33": 0.31048, + "34": 0.30963, + "35": 0.3029, + "36": 0.2968, + "37": 0.30603, + "38": 0.2963, + "39": 0.29696, + "40": 0.29692, + "41": 0.29566, + "42": 0.30692, + "43": 0.30493, + "44": 0.3071, + "45": 0.30615, + "46": 0.30053, + "47": 0.30066, + "48": 0.3008, + "49": 0.30036, + "50": 0.30389 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..49989195204 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 14.43366, + "2": 0.6497, + "3": 0.5625, + "4": 0.55843, + "5": 0.55295, + "6": 0.54999, + "7": 0.54892, + "8": 0.55625, + "9": 0.55227, + "10": 0.54797, + "11": 0.55219, + "12": 0.54869, + "13": 0.54779, + "14": 0.5432, + "15": 0.54305, + "16": 0.543, + "17": 0.54209, + "18": 0.54256, + "19": 0.53999, + "20": 0.5409, + "21": 0.54134, + "22": 0.54162, + "23": 0.54166, + "24": 0.5413, + "25": 0.54051, + "26": 0.55075, + "27": 0.54052, + "28": 0.54042, + "29": 0.54114, + "30": 0.54107, + "31": 0.54047, + "32": 0.53995, + "33": 0.54199, + "34": 0.54111, + "35": 0.54059, + "36": 0.54769, + "37": 0.5452, + "38": 0.54424, + "39": 0.54297, + "40": 0.54284, + "41": 0.54564, + "42": 0.55038, + "43": 0.55375, + "44": 0.54715, + "45": 0.55075, + "46": 0.55076, + "47": 0.55122, + "48": 0.55019, + "49": 0.55033, + "50": 0.54833 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json new file mode 100644 index 00000000000..446e0f113fc --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79175, + "2": 10.80907, + "3": 10.81011, + "4": 10.78146, + "5": 10.82288, + "6": 10.84057, + "7": 10.81192, + "8": 10.80005, + "9": 10.81667, + "10": 10.7688, + "11": 10.8618, + "12": 10.84042, + "13": 10.84452, + "14": 10.86421, + "15": 10.79157, + "16": 10.78199, + "17": 10.75122, + "18": 10.79446, + "19": 10.79523, + "20": 10.71001, + "21": 10.68811, + "22": 10.53736, + "23": 10.7066, + "24": 10.58865, + "25": 10.54662, + "26": 10.59492, + "27": 10.62142, + "28": 10.5969, + "29": 10.60036, + "30": 10.39407, + "31": 10.12951, + "32": 10.49684, + "33": 10.48779, + "34": 10.24347, + "35": 10.30461, + "36": 10.26056, + "37": 10.38859, + "38": 10.24848, + "39": 10.43799, + "40": 10.13303, + "41": 10.18651, + "42": 10.25823, + "43": 9.892, + "44": 10.02576, + "45": 9.90015, + "46": 9.88387, + "47": 10.19565, + "48": 9.91255, + "49": 9.60147, + "50": 9.97874 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5656.0, + "2": 6018.0, + "3": 5790.0, + "4": 5941.0, + "5": 6476.0, + "6": 6653.0, + "7": 6287.0, + "8": 5875.0, + "9": 6239.0, + "10": 5453.0, + "11": 6936.0, + "12": 6711.0, + "13": 6655.0, + "14": 6814.0, + "15": 6233.0, + "16": 6533.0, + "17": 6397.0, + "18": 6112.0, + "19": 6678.0, + "20": 5837.0, + "21": 6403.0, + "22": 5715.0, + "23": 6744.0, + "24": 6051.0, + "25": 5811.0, + "26": 6104.0, + "27": 6484.0, + "28": 6884.0, + "29": 7253.0, + "30": 6047.0, + "31": 5593.0, + "32": 6625.0, + "33": 7054.0, + "34": 6104.0, + "35": 6712.0, + "36": 6684.0, + "37": 7523.0, + "38": 7273.0, + "39": 7620.0, + "40": 7062.0, + "41": 6895.0, + "42": 7426.0, + "43": 6713.0, + "44": 6664.0, + "45": 6681.0, + "46": 6923.0, + "47": 7705.0, + "48": 7248.0, + "49": 7331.0, + "50": 7527.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 462408192.0, + "2": 462406144.0, + "3": 462409728.0, + "4": 462406144.0, + "5": 462407680.0, + "6": 462408192.0, + "7": 462410752.0, + "8": 462410752.0, + "9": 462407168.0, + "10": 462410240.0, + "11": 462408192.0, + "12": 462408192.0, + "13": 462408704.0, + "14": 462409728.0, + "15": 462409728.0, + "16": 462407168.0, + "17": 462408704.0, + "18": 462408704.0, + "19": 462408704.0, + "20": 462408704.0, + "21": 462406144.0, + "22": 462412800.0, + "23": 462409216.0, + "24": 462408704.0, + "25": 462406144.0, + "26": 462410240.0, + "27": 462405120.0, + "28": 462408192.0, + "29": 462407168.0, + "30": 462406144.0, + "31": 462413312.0, + "32": 462408704.0, + "33": 462409216.0, + "34": 462406144.0, + "35": 462410240.0, + "36": 462407168.0, + "37": 462409728.0, + "38": 462408192.0, + "39": 462408192.0, + "40": 462407680.0, + "41": 462411264.0, + "42": 462409728.0, + "43": 462411264.0, + "44": 462407680.0, + "45": 462408704.0, + "46": 462410752.0, + "47": 462407680.0, + "48": 462408192.0, + "49": 462409728.0, + "50": 462409216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1033451008.0, + "2": 1197371392.0, + "3": 1197371392.0, + "4": 1197880320.0, + "5": 1197880320.0, + "6": 1197880320.0, + "7": 1197880320.0, + "8": 1197880320.0, + "9": 1197965824.0, + "10": 1197965824.0, + "11": 1197965824.0, + "12": 1197965824.0, + "13": 1197965824.0, + "14": 1197965824.0, + "15": 1197965824.0, + "16": 1197965824.0, + "17": 1197965824.0, + "18": 1197965824.0, + "19": 1197965824.0, + "20": 1197965824.0, + "21": 1197965824.0, + "22": 1198112768.0, + "23": 1198112768.0, + "24": 1198112768.0, + "25": 1198112768.0, + "26": 1198112768.0, + "27": 1198112768.0, + "28": 1198112768.0, + "29": 1198112768.0, + "30": 1198112768.0, + "31": 1198112768.0, + "32": 1198112768.0, + "33": 1198112768.0, + "34": 1198112768.0, + "35": 1198112768.0, + "36": 1198112768.0, + "37": 1198112768.0, + "38": 1198112768.0, + "39": 1198112768.0, + "40": 1198333440.0, + "41": 1198333440.0, + "42": 1198333440.0, + "43": 1198444032.0, + "44": 1198444032.0, + "45": 1198444032.0, + "46": 1198444032.0, + "47": 1198444032.0, + "48": 1198444032.0, + "49": 1198444032.0, + "50": 1198444032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12.43217, + "2": 0.74038, + "3": 0.57793, + "4": 0.56188, + "5": 0.5615, + "6": 0.56664, + "7": 0.56245, + "8": 0.56336, + "9": 0.55994, + "10": 0.55792, + "11": 0.55747, + "12": 0.5609, + "13": 0.55976, + "14": 0.55747, + "15": 0.55815, + "16": 0.56073, + "17": 0.56435, + "18": 0.56034, + "19": 0.56151, + "20": 0.55653, + "21": 0.55567, + "22": 0.55628, + "23": 0.55513, + "24": 0.55783, + "25": 0.55752, + "26": 0.55622, + "27": 0.55558, + "28": 0.55732, + "29": 0.55679, + "30": 0.55683, + "31": 0.55859, + "32": 0.55549, + "33": 0.55698, + "34": 0.55654, + "35": 0.55834, + "36": 0.55776, + "37": 0.55524, + "38": 0.55487, + "39": 0.55761, + "40": 0.55611, + "41": 0.55671, + "42": 0.55782, + "43": 0.55892, + "44": 0.55946, + "45": 0.55729, + "46": 0.55907, + "47": 0.56002, + "48": 0.56693, + "49": 0.56346, + "50": 0.55665 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json new file mode 100644 index 00000000000..da27aa14ba7 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82004, + "2": 10.8392, + "3": 10.81124, + "4": 10.81983, + "5": 10.84794, + "6": 10.8608, + "7": 10.84085, + "8": 10.84432, + "9": 10.8504, + "10": 10.79461, + "11": 10.85658, + "12": 10.84848, + "13": 10.86929, + "14": 10.8667, + "15": 10.82911, + "16": 10.81111, + "17": 10.79027, + "18": 10.80981, + "19": 10.81143, + "20": 10.73175, + "21": 10.71285, + "22": 10.58199, + "23": 10.72, + "24": 10.61704, + "25": 10.57964, + "26": 10.63372, + "27": 10.6365, + "28": 10.60641, + "29": 10.61561, + "30": 10.40859, + "31": 10.17068, + "32": 10.49958, + "33": 10.4963, + "34": 10.25574, + "35": 10.31503, + "36": 10.28536, + "37": 10.38742, + "38": 10.24676, + "39": 10.44249, + "40": 10.14367, + "41": 10.19116, + "42": 10.25654, + "43": 9.90671, + "44": 10.02653, + "45": 9.914, + "46": 9.89613, + "47": 10.18885, + "48": 9.92993, + "49": 9.61419, + "50": 9.97565 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 12826.0, + "2": 14613.0, + "3": 14549.0, + "4": 13422.0, + "5": 15951.0, + "6": 16055.0, + "7": 15208.0, + "8": 12944.0, + "9": 15110.0, + "10": 12611.0, + "11": 16586.0, + "12": 14954.0, + "13": 15925.0, + "14": 16182.0, + "15": 14834.0, + "16": 16023.0, + "17": 15486.0, + "18": 15116.0, + "19": 15584.0, + "20": 13675.0, + "21": 13873.0, + "22": 12917.0, + "23": 16766.0, + "24": 13924.0, + "25": 13129.0, + "26": 14794.0, + "27": 15169.0, + "28": 16393.0, + "29": 16719.0, + "30": 14652.0, + "31": 13126.0, + "32": 15987.0, + "33": 17372.0, + "34": 14206.0, + "35": 15183.0, + "36": 15837.0, + "37": 17507.0, + "38": 16617.0, + "39": 17712.0, + "40": 16971.0, + "41": 16795.0, + "42": 17304.0, + "43": 15578.0, + "44": 15564.0, + "45": 16188.0, + "46": 17443.0, + "47": 19238.0, + "48": 16575.0, + "49": 16273.0, + "50": 18998.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 629738496.0, + "2": 629705216.0, + "3": 629710336.0, + "4": 629699584.0, + "5": 629732864.0, + "6": 629703168.0, + "7": 629718528.0, + "8": 629722624.0, + "9": 629763072.0, + "10": 629733888.0, + "11": 629810176.0, + "12": 629705728.0, + "13": 629848576.0, + "14": 629702144.0, + "15": 629870592.0, + "16": 629805568.0, + "17": 629698048.0, + "18": 629731328.0, + "19": 629798912.0, + "20": 629752832.0, + "21": 629716480.0, + "22": 629699584.0, + "23": 629705216.0, + "24": 629736448.0, + "25": 629699584.0, + "26": 629736960.0, + "27": 629704192.0, + "28": 629750272.0, + "29": 629728256.0, + "30": 629933568.0, + "31": 629847040.0, + "32": 629700096.0, + "33": 629703168.0, + "34": 629752832.0, + "35": 629725696.0, + "36": 629724160.0, + "37": 629702656.0, + "38": 629704192.0, + "39": 629733888.0, + "40": 629749760.0, + "41": 629700096.0, + "42": 629729280.0, + "43": 629699072.0, + "44": 629769728.0, + "45": 629713920.0, + "46": 629804544.0, + "47": 629719552.0, + "48": 629843456.0, + "49": 630007296.0, + "50": 629703168.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1847654400.0, + "2": 2077632000.0, + "3": 2078750208.0, + "4": 2078750208.0, + "5": 2079077888.0, + "6": 2079077888.0, + "7": 2080535040.0, + "8": 2080535040.0, + "9": 2080535040.0, + "10": 2080535040.0, + "11": 2080535040.0, + "12": 2080535040.0, + "13": 2080535040.0, + "14": 2080535040.0, + "15": 2080535040.0, + "16": 2080535040.0, + "17": 2080535040.0, + "18": 2080535040.0, + "19": 2080535040.0, + "20": 2080535040.0, + "21": 2080535040.0, + "22": 2080535040.0, + "23": 2080535040.0, + "24": 2080535040.0, + "25": 2080535040.0, + "26": 2080535040.0, + "27": 2080535040.0, + "28": 2080535040.0, + "29": 2080535040.0, + "30": 2080535040.0, + "31": 2080535040.0, + "32": 2080535040.0, + "33": 2080535040.0, + "34": 2080535040.0, + "35": 2080535040.0, + "36": 2080535040.0, + "37": 2080535040.0, + "38": 2080535040.0, + "39": 2080535040.0, + "40": 2080535040.0, + "41": 2080535040.0, + "42": 2080535040.0, + "43": 2080535040.0, + "44": 2080535040.0, + "45": 2080535040.0, + "46": 2080535040.0, + "47": 2080535040.0, + "48": 2080535040.0, + "49": 2080535040.0, + "50": 2080535040.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 7.62659, + "2": 0.38341, + "3": 0.33635, + "4": 0.34116, + "5": 0.34072, + "6": 0.3389, + "7": 0.34155, + "8": 0.34198, + "9": 0.33754, + "10": 0.33203, + "11": 0.33254, + "12": 0.33264, + "13": 0.33319, + "14": 0.33151, + "15": 0.33141, + "16": 0.33262, + "17": 0.33144, + "18": 0.33178, + "19": 0.33305, + "20": 0.33314, + "21": 0.3316, + "22": 0.33139, + "23": 0.3341, + "24": 0.33144, + "25": 0.33162, + "26": 0.33101, + "27": 0.33291, + "28": 0.33159, + "29": 0.33154, + "30": 0.33361, + "31": 0.33923, + "32": 0.32883, + "33": 0.35272, + "34": 0.33492, + "35": 0.33596, + "36": 0.33769, + "37": 0.33479, + "38": 0.33255, + "39": 0.33237, + "40": 0.33371, + "41": 0.33213, + "42": 0.33386, + "43": 0.33396, + "44": 0.33185, + "45": 0.33435, + "46": 0.33282, + "47": 0.33161, + "48": 0.33169, + "49": 0.33131, + "50": 0.32971 + } + } +} \ No newline at end of file diff --git a/uv.lock b/uv.lock index 5451e635d0c..3de5e29b62b 100644 --- a/uv.lock +++ b/uv.lock @@ -300,12 +300,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] -[[package]] -name = "asciitree" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", size = 3951, upload-time = "2016-09-05T19:10:42.681Z" } - [[package]] name = "astroid" version = "3.2.4" @@ -986,82 +980,6 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] -[[package]] -name = "crc32c" -version = "2.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/a0/28b4686a8db0bb0f77970f4c6ccede90d1d5740a1d4b4703bd54c3e75655/crc32c-2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2c0f4eb01fe7c0a3e3f973a418e04d52101bb077dd77626fd80c658ec60aaf95", size = 66321, upload-time = "2025-10-17T06:18:53.543Z" }, - { url = "https://files.pythonhosted.org/packages/76/1f/1697f5b8b770f715ed9b264d79e36b4f77ae0527f81f3c749ef08937a32e/crc32c-2.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6baefcfbca82b1a9678455416da24f18629769a76920c640d5a538620a7d12bb", size = 62985, upload-time = "2025-10-17T06:18:54.97Z" }, - { url = "https://files.pythonhosted.org/packages/e0/e5/333cfa5ffa8d5779733aced2b984b5e5139b4a8ceaa2c6bc563e9a1092f3/crc32c-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7f959fcf6c5aad1c4a653ee1a50f05760dab1d1c35d98ec4d7f0f68643f7612", size = 61517, upload-time = "2025-10-17T06:18:55.795Z" }, - { url = "https://files.pythonhosted.org/packages/e1/d8/362a009e8140dd926a153b44d56753e3aa7cb50aca243779a84adadbff11/crc32c-2.8-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bb678507a4e4cf3f0506607b046ecc4ed1c58a19e08a3fb3c2d25441c480bf1", size = 79385, upload-time = "2025-10-17T06:18:56.598Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0d4ea3aa71ffb15f1285669d23024cc40779388ce32157d339dc2584491c/crc32c-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a16f7ffa4c242a909558565567cbba95148603717b53538ea299c98da68e7a9", size = 80965, upload-time = "2025-10-17T06:18:57.384Z" }, - { url = "https://files.pythonhosted.org/packages/20/44/d77657aaca4a2c0283f2356a3da6f8e91b003567bb8f09daaf540cbf192f/crc32c-2.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0184369aad562d801f91f454c81f56b9ecb966f6b96684c4d6cf82fc8741d2ad", size = 79993, upload-time = "2025-10-17T06:18:58.503Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c0/07017a93ebf85d9408028b7e03ef96d5c6bfb14cb77cfe90d35eedcc1501/crc32c-2.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86d2eeb5f0189bd803720abe7387019328ea34c4acde62999e5723f789bc316b", size = 79243, upload-time = "2025-10-17T06:18:59.273Z" }, - { url = "https://files.pythonhosted.org/packages/c7/1a/b3c5ac4cf2fd1f82395173d0bd8e1a15d09f0bc1eccdf10ea7f8caaccd67/crc32c-2.8-cp310-cp310-win32.whl", hash = "sha256:51da61904a9e753780a2e6011885677d601db1fa840be4b68799643a113e6f08", size = 64888, upload-time = "2025-10-17T06:19:00.089Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f2/60c45fc7bb2221d3c93c7a872e921be591f40d45228fe46f879b1d8c0424/crc32c-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:b2d6a1f2500daaf2e4b08f97ad0349aa2eff5faaaa5fd3350314a26eade334cd", size = 66639, upload-time = "2025-10-17T06:19:00.974Z" }, - { url = "https://files.pythonhosted.org/packages/dc/0b/5e03b22d913698e9cc563f39b9f6bbd508606bf6b8e9122cd6bf196b87ea/crc32c-2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e560a97fbb96c9897cb1d9b5076ef12fc12e2e25622530a1afd0de4240f17e1f", size = 66329, upload-time = "2025-10-17T06:19:01.771Z" }, - { url = "https://files.pythonhosted.org/packages/6b/38/2fe0051ffe8c6a650c8b1ac0da31b8802d1dbe5fa40a84e4b6b6f5583db5/crc32c-2.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6762d276d90331a490ef7e71ffee53b9c0eb053bd75a272d786f3b08d3fe3671", size = 62988, upload-time = "2025-10-17T06:19:02.953Z" }, - { url = "https://files.pythonhosted.org/packages/3e/30/5837a71c014be83aba1469c58820d287fc836512a0cad6b8fdd43868accd/crc32c-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:60670569f5ede91e39f48fb0cb4060e05b8d8704dd9e17ede930bf441b2f73ef", size = 61522, upload-time = "2025-10-17T06:19:03.796Z" }, - { url = "https://files.pythonhosted.org/packages/ca/29/63972fc1452778e2092ae998c50cbfc2fc93e3fa9798a0278650cd6169c5/crc32c-2.8-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:711743da6ccc70b3c6718c328947b0b6f34a1fe6a6c27cc6c1d69cc226bf70e9", size = 80200, upload-time = "2025-10-17T06:19:04.617Z" }, - { url = "https://files.pythonhosted.org/packages/cb/3a/60eb49d7bdada4122b3ffd45b0df54bdc1b8dd092cda4b069a287bdfcff4/crc32c-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eb4094a2054774f13b26f21bf56792bb44fa1fcee6c6ad099387a43ffbfb4fa", size = 81757, upload-time = "2025-10-17T06:19:05.496Z" }, - { url = "https://files.pythonhosted.org/packages/f5/63/6efc1b64429ef7d23bd58b75b7ac24d15df327e3ebbe9c247a0f7b1c2ed1/crc32c-2.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fff15bf2bd3e95780516baae935ed12be88deaa5ebe6143c53eb0d26a7bdc7b7", size = 80830, upload-time = "2025-10-17T06:19:06.621Z" }, - { url = "https://files.pythonhosted.org/packages/e1/eb/0ae9f436f8004f1c88f7429e659a7218a3879bd11a6b18ed1257aad7e98b/crc32c-2.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c0e11e3826668121fa53e0745635baf5e4f0ded437e8ff63ea56f38fc4f970a", size = 80095, upload-time = "2025-10-17T06:19:07.381Z" }, - { url = "https://files.pythonhosted.org/packages/9e/81/4afc9d468977a4cd94a2eb62908553345009a7c0d30e74463a15d4b48ec3/crc32c-2.8-cp311-cp311-win32.whl", hash = "sha256:38f915336715d1f1353ab07d7d786f8a789b119e273aea106ba55355dfc9101d", size = 64886, upload-time = "2025-10-17T06:19:08.497Z" }, - { url = "https://files.pythonhosted.org/packages/d6/e8/94e839c9f7e767bf8479046a207afd440a08f5c59b52586e1af5e64fa4a0/crc32c-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:60e0a765b1caab8d31b2ea80840639253906a9351d4b861551c8c8625ea20f86", size = 66639, upload-time = "2025-10-17T06:19:09.338Z" }, - { url = "https://files.pythonhosted.org/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab", size = 66379, upload-time = "2025-10-17T06:19:10.115Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68", size = 63042, upload-time = "2025-10-17T06:19:10.946Z" }, - { url = "https://files.pythonhosted.org/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2", size = 61528, upload-time = "2025-10-17T06:19:11.768Z" }, - { url = "https://files.pythonhosted.org/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2", size = 80028, upload-time = "2025-10-17T06:19:12.554Z" }, - { url = "https://files.pythonhosted.org/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1", size = 81531, upload-time = "2025-10-17T06:19:13.406Z" }, - { url = "https://files.pythonhosted.org/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e", size = 80608, upload-time = "2025-10-17T06:19:14.223Z" }, - { url = "https://files.pythonhosted.org/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c", size = 79886, upload-time = "2025-10-17T06:19:15.345Z" }, - { url = "https://files.pythonhosted.org/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76", size = 64896, upload-time = "2025-10-17T06:19:16.457Z" }, - { url = "https://files.pythonhosted.org/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2", size = 66645, upload-time = "2025-10-17T06:19:17.235Z" }, - { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, - { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, - { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, - { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, - { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, - { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, - { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, - { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, - { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, - { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, - { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, - { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, - { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, - { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, - { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, - { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, - { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, - { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, - { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, - { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, - { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, - { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, - { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, - { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, - { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, - { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, - { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, - { url = "https://files.pythonhosted.org/packages/a7/1d/dd926c68eb8aac8b142a1a10b8eb62d95212c1cf81775644373fe7cceac2/crc32c-2.8-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5833f4071da7ea182c514ba17d1eee8aec3c5be927d798222fbfbbd0f5eea02c", size = 62345, upload-time = "2025-10-17T06:20:09.39Z" }, - { url = "https://files.pythonhosted.org/packages/51/be/803404e5abea2ef2c15042edca04bbb7f625044cca879e47f186b43887c2/crc32c-2.8-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1dc4da036126ac07b39dd9d03e93e585ec615a2ad28ff12757aef7de175295a8", size = 61229, upload-time = "2025-10-17T06:20:10.236Z" }, - { url = "https://files.pythonhosted.org/packages/fc/3a/00cc578cd27ed0b22c9be25cef2c24539d92df9fa80ebd67a3fc5419724c/crc32c-2.8-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:15905fa78344654e241371c47e6ed2411f9eeb2b8095311c68c88eccf541e8b4", size = 64108, upload-time = "2025-10-17T06:20:11.072Z" }, - { url = "https://files.pythonhosted.org/packages/6b/bc/0587ef99a1c7629f95dd0c9d4f3d894de383a0df85831eb16c48a6afdae4/crc32c-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c596f918688821f796434e89b431b1698396c38bf0b56de873621528fe3ecb1e", size = 64815, upload-time = "2025-10-17T06:20:11.919Z" }, - { url = "https://files.pythonhosted.org/packages/73/42/94f2b8b92eae9064fcfb8deef2b971514065bd606231f8857ff8ae02bebd/crc32c-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8d23c4fe01b3844cb6e091044bc1cebdef7d16472e058ce12d9fadf10d2614af", size = 66659, upload-time = "2025-10-17T06:20:12.766Z" }, -] - [[package]] name = "cryptography" version = "42.0.8" @@ -1182,8 +1100,8 @@ name = "deprecated" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ @@ -1231,18 +1149,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "ebmlite" version = "3.4.1" @@ -1303,15 +1209,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/cc/1c33d05f62c9349bb80dfe789cc9a7409bdfb337a63fa347fd651d25294a/fastapi-0.120.2-py3-none-any.whl", hash = "sha256:bedcf2c14240e43d56cb9a339b32bcf15104fe6b5897c0222603cb7ec416c8eb", size = 108383, upload-time = "2025-10-29T13:47:32.978Z" }, ] -[[package]] -name = "fasteners" -version = "0.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz", hash = "sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", size = 25087, upload-time = "2025-08-11T10:19:37.785Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl", hash = "sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", size = 18702, upload-time = "2025-08-11T10:19:35.716Z" }, -] - [[package]] name = "fiddle" version = "0.3.0" @@ -1814,7 +1711,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/08/c1395a292bb23fd03bdf572a1357c5a733d3eecbab877641ceacab23db6e/importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580", size = 55767, upload-time = "2025-01-20T22:21:30.429Z" } wheels = [ @@ -1826,35 +1723,23 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "zipp", marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "zipp", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -2280,16 +2165,23 @@ dev = [ { name = "wget" }, ] lts = [ + { name = "av" }, + { name = "causal-conv1d" }, { name = "einops" }, + { name = "flashinfer-python" }, + { name = "mamba-ssm" }, + { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-lts'" }, + { name = "multi-storage-client" }, + { name = "nv-grouped-gemm" }, { name = "nvtx" }, + { name = "onnxscript", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnxscript", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "opentelemetry-api" }, { name = "setuptools" }, { name = "tensorstore", version = "0.1.74", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tqdm" }, - { name = "transformers" }, { name = "wget" }, - { name = "zarr", version = "2.18.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "zarr", version = "3.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] mlm = [ { name = "flask-restful" }, @@ -2347,46 +2239,53 @@ test = [ { name = "pytest-random-order" }, { name = "pyyaml" }, { name = "tensorboard" }, - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, ] [package.metadata] requires-dist = [ { name = "av", marker = "extra == 'dev'", specifier = "<16.0.0" }, + { name = "av", marker = "extra == 'lts'", specifier = "<16.0.0" }, { name = "causal-conv1d", marker = "extra == 'dev'", specifier = "~=1.5" }, + { name = "causal-conv1d", marker = "extra == 'lts'", specifier = "~=1.5" }, { name = "einops", marker = "extra == 'dev'", specifier = "~=0.8" }, - { name = "einops", marker = "extra == 'lts'" }, + { name = "einops", marker = "extra == 'lts'", specifier = "~=0.8" }, { name = "flashinfer-python", marker = "extra == 'dev'" }, + { name = "flashinfer-python", marker = "extra == 'lts'" }, { name = "flask-restful", marker = "extra == 'mlm'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, + { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, + { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'lts'", specifier = "~=6.0" }, { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, + { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy", specifier = "<2.0.0" }, { name = "nv-grouped-gemm", marker = "extra == 'dev'", specifier = "~=1.1" }, + { name = "nv-grouped-gemm", marker = "extra == 'lts'", specifier = "~=1.1" }, { name = "nvidia-modelopt", extras = ["torch"], marker = "sys_platform != 'darwin' and extra == 'dev'" }, - { name = "nvidia-resiliency-ext", marker = "extra == 'dev'", specifier = ">=0.4.0a0,<0.5.0" }, + { name = "nvidia-resiliency-ext", marker = "extra == 'dev'" }, { name = "nvtx", marker = "extra == 'dev'", specifier = "~=0.2" }, - { name = "nvtx", marker = "extra == 'lts'" }, + { name = "nvtx", marker = "extra == 'lts'", specifier = "~=0.2" }, { name = "onnxscript", marker = "extra == 'dev'" }, + { name = "onnxscript", marker = "extra == 'lts'" }, { name = "opentelemetry-api", marker = "extra == 'dev'", specifier = "~=1.33.1" }, + { name = "opentelemetry-api", marker = "extra == 'lts'", specifier = "~=1.33.1" }, { name = "packaging", specifier = ">=24.2" }, { name = "sentencepiece", marker = "extra == 'mlm'" }, { name = "setuptools", marker = "extra == 'dev'", specifier = "<80.0.0" }, { name = "setuptools", marker = "extra == 'lts'", specifier = "<80.0.0" }, { name = "tensorstore", marker = "extra == 'dev'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, - { name = "tensorstore", marker = "extra == 'lts'", specifier = "!=0.1.46,!=0.1.72" }, + { name = "tensorstore", marker = "extra == 'lts'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, { name = "tiktoken", marker = "extra == 'mlm'" }, { name = "torch" }, { name = "tqdm", marker = "extra == 'dev'" }, { name = "tqdm", marker = "extra == 'lts'" }, { name = "transformer-engine", extras = ["pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.9" }, - { name = "transformers", marker = "extra == 'lts'" }, { name = "transformers", marker = "extra == 'mlm'" }, { name = "wandb", marker = "extra == 'mlm'" }, { name = "wget", marker = "extra == 'dev'" }, { name = "wget", marker = "extra == 'lts'" }, - { name = "zarr", marker = "extra == 'lts'" }, ] provides-extras = ["mlm", "dev", "lts"] @@ -2939,81 +2838,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/90/81ac364ef94209c100e12579629dc92bf7a709a84af32f8c551b02c07e94/nltk-3.9.2-py3-none-any.whl", hash = "sha256:1e209d2b3009110635ed9709a67a1a3e33a10f799490fa71cf4bec218c11c88a", size = 1513404, upload-time = "2025-10-01T07:19:21.648Z" }, ] -[[package]] -name = "numcodecs" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/56/8895a76abe4ec94ebd01eeb6d74f587bc4cddd46569670e1402852a5da13/numcodecs-0.13.1.tar.gz", hash = "sha256:a3cf37881df0898f3a9c0d4477df88133fe85185bffe57ba31bcc2fa207709bc", size = 5955215, upload-time = "2024-10-09T16:28:00.188Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/c0/6d72cde772bcec196b7188731d41282993b2958440f77fdf0db216f722da/numcodecs-0.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:96add4f783c5ce57cc7e650b6cac79dd101daf887c479a00a29bc1487ced180b", size = 1580012, upload-time = "2024-10-09T16:27:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/94/1d/f81fc1fa9210bbea97258242393a1f9feab4f6d8fb201f81f76003005e4b/numcodecs-0.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:237b7171609e868a20fd313748494444458ccd696062f67e198f7f8f52000c15", size = 1176919, upload-time = "2024-10-09T16:27:21.634Z" }, - { url = "https://files.pythonhosted.org/packages/16/e4/b9ec2f4dfc34ecf724bc1beb96a9f6fa9b91801645688ffadacd485089da/numcodecs-0.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96e42f73c31b8c24259c5fac6adba0c3ebf95536e37749dc6c62ade2989dca28", size = 8625842, upload-time = "2024-10-09T16:27:24.168Z" }, - { url = "https://files.pythonhosted.org/packages/fe/90/299952e1477954ec4f92813fa03e743945e3ff711bb4f6c9aace431cb3da/numcodecs-0.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:eda7d7823c9282e65234731fd6bd3986b1f9e035755f7fed248d7d366bb291ab", size = 828638, upload-time = "2024-10-09T16:27:27.063Z" }, - { url = "https://files.pythonhosted.org/packages/f0/78/34b8e869ef143e88d62e8231f4dbfcad85e5c41302a11fc5bd2228a13df5/numcodecs-0.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2eda97dd2f90add98df6d295f2c6ae846043396e3d51a739ca5db6c03b5eb666", size = 1580199, upload-time = "2024-10-09T16:27:29.336Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cf/f70797d86bb585d258d1e6993dced30396f2044725b96ce8bcf87a02be9c/numcodecs-0.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2a86f5367af9168e30f99727ff03b27d849c31ad4522060dde0bce2923b3a8bc", size = 1177203, upload-time = "2024-10-09T16:27:31.011Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b5/d14ad69b63fde041153dfd05d7181a49c0d4864de31a7a1093c8370da957/numcodecs-0.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233bc7f26abce24d57e44ea8ebeb5cd17084690b4e7409dd470fdb75528d615f", size = 8868743, upload-time = "2024-10-09T16:27:32.833Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/27a7b5af0b33f6d61e198faf177fbbf3cb83ff10d9d1a6857b7efc525ad5/numcodecs-0.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:796b3e6740107e4fa624cc636248a1580138b3f1c579160f260f76ff13a4261b", size = 829603, upload-time = "2024-10-09T16:27:35.415Z" }, - { url = "https://files.pythonhosted.org/packages/37/3a/bc09808425e7d3df41e5fc73fc7a802c429ba8c6b05e55f133654ade019d/numcodecs-0.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5195bea384a6428f8afcece793860b1ab0ae28143c853f0b2b20d55a8947c917", size = 1575806, upload-time = "2024-10-09T16:27:37.804Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cc/dc74d0bfdf9ec192332a089d199f1e543e747c556b5659118db7a437dcca/numcodecs-0.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3501a848adaddce98a71a262fee15cd3618312692aa419da77acd18af4a6a3f6", size = 1178233, upload-time = "2024-10-09T16:27:40.169Z" }, - { url = "https://files.pythonhosted.org/packages/d4/ce/434e8e3970b8e92ae9ab6d9db16cb9bc7aa1cd02e17c11de6848224100a1/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53", size = 8857827, upload-time = "2024-10-09T16:27:42.743Z" }, - { url = "https://files.pythonhosted.org/packages/83/e7/1d8b1b266a92f9013c755b1c146c5ad71a2bff147ecbc67f86546a2e4d6a/numcodecs-0.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:e5db4824ebd5389ea30e54bc8aeccb82d514d28b6b68da6c536b8fa4596f4bca", size = 826539, upload-time = "2024-10-09T16:27:44.808Z" }, - { url = "https://files.pythonhosted.org/packages/83/8b/06771dead2cc4a8ae1ea9907737cf1c8d37a323392fa28f938a586373468/numcodecs-0.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a60d75179fd6692e301ddfb3b266d51eb598606dcae7b9fc57f986e8d65cb43", size = 1571660, upload-time = "2024-10-09T16:27:47.125Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ea/d925bf85f92dfe4635356018da9fe4bfecb07b1c72f62b01c1bc47f936b1/numcodecs-0.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f593c7506b0ab248961a3b13cb148cc6e8355662ff124ac591822310bc55ecf", size = 1169925, upload-time = "2024-10-09T16:27:49.512Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d6/643a3839d571d8e439a2c77dc4b0b8cab18d96ac808e4a81dbe88e959ab6/numcodecs-0.13.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d3071465f03522e776a31045ddf2cfee7f52df468b977ed3afdd7fe5869701", size = 8814257, upload-time = "2024-10-09T16:27:52.059Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c5/f3e56bc9b4e438a287fff738993d6d11abef368c0328a612ac2842ba9fca/numcodecs-0.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:90d3065ae74c9342048ae0046006f99dcb1388b7288da5a19b3bddf9c30c3176", size = 821887, upload-time = "2024-10-09T16:27:55.039Z" }, -] - -[[package]] -name = "numcodecs" -version = "0.16.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/48/6188e359b90a9d8a1850f2bc888c023e66f4a8b2b496820babbea414f008/numcodecs-0.16.3.tar.gz", hash = "sha256:53d705865faaf0a7927c973af3777532001c8fbb653de119c1e844608614d799", size = 6275704, upload-time = "2025-09-18T18:54:57.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/cc/917a85972537498f2bbd7914047efc98babc8667587ceb9dcb228378978a/numcodecs-0.16.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:95c9f2a49bef10cf91ad614a761cba9bfe96656b60c12540e1080de5d909b4ca", size = 1642356, upload-time = "2025-09-18T18:54:36.402Z" }, - { url = "https://files.pythonhosted.org/packages/3b/6a/64c25a089e8537441fe67c09ecb7f3f7fb5d98cd04faf01f605d43aca41c/numcodecs-0.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2afe73d5ebaf9ca0cd5c83aad945da80d29a33d860a80d43a7248491d8813ff", size = 1169186, upload-time = "2025-09-18T18:54:37.838Z" }, - { url = "https://files.pythonhosted.org/packages/d8/a0/0de627baeb43e2045a3d4b3de99bf8b69af329a33df1ed4cda468d70c1fb/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:913f08194d82dcb37594e6705e6d4ae6ccd4b6571500b832fb3e4a155de1dfe8", size = 8341668, upload-time = "2025-09-18T18:54:39.444Z" }, - { url = "https://files.pythonhosted.org/packages/b6/0f/49d1f74a216149240c4b9403218111f11670bd11af0919fda357bb056bf2/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a7f1cae9eb18b85709af46570bf9c60056e7155c4c8f610e8080c68124d0e5", size = 8866611, upload-time = "2025-09-18T18:54:41.168Z" }, - { url = "https://files.pythonhosted.org/packages/aa/51/03aece765108fe247717105b5131856546e5428f22a56a14ffdebd017424/numcodecs-0.16.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7bb7f2c46eb7ec8a1c5f8d8fe1a72c222256dd6d6df5af9eaac7a6b905f3575", size = 806787, upload-time = "2025-09-18T18:54:42.78Z" }, - { url = "https://files.pythonhosted.org/packages/0d/78/e4b34803a3aa1d0769919695de4b133266c18c80c474d32ebc462fa1a9bd/numcodecs-0.16.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c77454d92941a335d148b0b822f5d4783103f392774d5d76283bbf7f21b49529", size = 1681108, upload-time = "2025-09-18T18:54:43.856Z" }, - { url = "https://files.pythonhosted.org/packages/25/cf/ca36f463b03a4097767d2a1c1b72f31810e8c6384e9449dd9b925203783c/numcodecs-0.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:270e7a33ee96bdf5c957acf25a2487002a233811a125a155c400c2f036b69c73", size = 1165589, upload-time = "2025-09-18T18:54:44.954Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ae/670260c3c4b5ed34a0674561355f3d4ce7fcbdf09a667e5bc841526d271c/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12f43fa4a347d1dba775c4506a1c9b15b90144c258433b81f79f1c1b1a990db5", size = 8316365, upload-time = "2025-09-18T18:54:46.073Z" }, - { url = "https://files.pythonhosted.org/packages/bb/fa/94e022419c751a60ff0f53642ebae5ef81ed3cc3640f958588e3ad3dc18d/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44869ef564a50aa545215c6a0d42ba5bbc34e9715523fb2336ada3d1fb2b331d", size = 8846228, upload-time = "2025-09-18T18:54:47.858Z" }, - { url = "https://files.pythonhosted.org/packages/71/60/f23733589f3e059bf8589508acd23ffeec230bdf179f138a54f5ab16e0a6/numcodecs-0.16.3-cp312-cp312-win_amd64.whl", hash = "sha256:9aae6996172ba10c5f5111b2998709071b5aeba6b58b1ee0b26b61ed6aa7f2f4", size = 806260, upload-time = "2025-09-18T18:54:49.41Z" }, - { url = "https://files.pythonhosted.org/packages/3c/d5/d3536d06ac1e5fb848a3186958204082b68b106364c9a3669652dd786731/numcodecs-0.16.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:947406b01c20f2ce7ce2e631e7f21b782e8a9d4b57b374a41c9e7b1341a8f3a2", size = 1677129, upload-time = "2025-09-18T18:54:50.5Z" }, - { url = "https://files.pythonhosted.org/packages/e1/fd/b0513a3428dc2b38ec85eea771703ae69c49f09b9650d6c44c9105c80073/numcodecs-0.16.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7cf50e351398a34b45817974c411527629e88937b7683695e276afd65da6ed6f", size = 1159058, upload-time = "2025-09-18T18:54:51.675Z" }, - { url = "https://files.pythonhosted.org/packages/98/05/b7c127283cfb154a97abb284363825401b69302d71a28608af66f73257cc/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7938502fcc060ed9543814f38ca67048b33d7bd2667756e36e6b1060455b17e", size = 8260987, upload-time = "2025-09-18T18:54:52.883Z" }, - { url = "https://files.pythonhosted.org/packages/ff/46/320d960aff884bc63abaaf846ffa3de4803e83e8070b6f84c5688464839c/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:010d628c95be1214536fb22c0df4ced58da954b404b1fcb25ddebf64e4a3f7f3", size = 8805295, upload-time = "2025-09-18T18:54:54.698Z" }, - { url = "https://files.pythonhosted.org/packages/31/ae/acc2e0f1f49ba32afa2174578f170673139248ef86f77e334f2619133867/numcodecs-0.16.3-cp313-cp313-win_amd64.whl", hash = "sha256:e83115e3c32de798c7b7164503e06aae9f9746c1cef564d029616eb44bd6cd90", size = 803204, upload-time = "2025-09-18T18:54:56.192Z" }, -] - -[package.optional-dependencies] -crc32c = [ - { name = "crc32c", marker = "python_full_version >= '3.11'" }, -] - [[package]] name = "numpy" version = "1.26.4" @@ -3057,59 +2881,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/05/79/87c45f32e661b25e0aaa1e325ba166511f57be5dff8f0fcabc12d3e73b64/nv_grouped_gemm-1.1.4.post6.tar.gz", hash = "sha256:dad6115f4b4ff7ceb0bc40ad44e923c13a24fc88cfe1e20b1a6b4c9cf24c445c", size = 26508, upload-time = "2025-10-10T18:52:29.508Z" } -[[package]] -name = "nvidia-cublas-cu12" -version = "12.8.4.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, - { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, - { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208, upload-time = "2025-03-07T01:53:30.535Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, - { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, - { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759, upload-time = "2025-03-07T01:51:11.355Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, - { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, - { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838, upload-time = "2025-03-07T01:52:13.483Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, - { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, - { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318, upload-time = "2025-03-07T01:51:01.794Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu12" -version = "9.10.2.21" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, - { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, - { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268, upload-time = "2025-06-06T21:55:18.114Z" }, -] - [[package]] name = "nvidia-cudnn-frontend" version = "1.15.0" @@ -3129,76 +2900,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/04/519fd6e3ea12fe7fe98c497c4d51f6c5c87763d02e90ea3102cef32a6ef1/nvidia_cudnn_frontend-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:7c8c6f12534b73b0cd55956c5e9419b7840a01e4c260837606112450ce1ca0d9", size = 1297324, upload-time = "2025-10-10T18:46:53.104Z" }, ] -[[package]] -name = "nvidia-cufft-cu12" -version = "11.3.3.83" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, - { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559, upload-time = "2025-03-07T01:53:57.106Z" }, -] - -[[package]] -name = "nvidia-cufile-cu12" -version = "1.13.1.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, -] - -[[package]] -name = "nvidia-curand-cu12" -version = "10.3.9.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, - { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, - { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309, upload-time = "2025-03-07T01:54:20.478Z" }, -] - -[[package]] -name = "nvidia-cusolver-cu12" -version = "11.7.3.90" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, - { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, - { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438, upload-time = "2025-03-07T01:54:39.898Z" }, -] - -[[package]] -name = "nvidia-cusparse-cu12" -version = "12.5.8.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, - { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404, upload-time = "2025-03-07T01:55:07.742Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu12" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, - { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911, upload-time = "2025-02-26T00:14:47.204Z" }, -] - [[package]] name = "nvidia-mathdx" version = "25.1.1" @@ -3240,44 +2941,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/a8/85034a33753a56ef120b931dda1183d98efef1010e379c54d3c3214a5048/nvidia_modelopt-0.37.0-py3-none-any.whl", hash = "sha256:3490b6d6aea3541aa5d475d81230fee627e2c16ff47bbab1cba4b80a1eb119a2", size = 831271, upload-time = "2025-10-08T18:37:03.951Z" }, ] -[[package]] -name = "nvidia-nccl-cu12" -version = "2.27.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/1c/857979db0ef194ca5e21478a0612bcdbbe59458d7694361882279947b349/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a", size = 322400625, upload-time = "2025-06-26T04:11:04.496Z" }, - { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, -] - -[[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710, upload-time = "2025-03-07T01:56:24.13Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu12" -version = "3.3.20" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9d/3dd98852568fb845ec1f7902c90a22b240fe1cbabda411ccedf2fd737b7b/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0", size = 124484616, upload-time = "2025-08-04T20:24:59.172Z" }, - { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, -] - -[[package]] -name = "nvidia-nvtx-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, - { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492, upload-time = "2025-03-07T01:52:24.69Z" }, -] - [[package]] name = "nvidia-resiliency-ext" version = "0.4.1" @@ -3351,8 +3014,8 @@ name = "onnx" version = "1.19.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy" }, { name = "protobuf" }, { name = "typing-extensions" }, @@ -5427,7 +5090,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -5711,30 +5374,15 @@ name = "torch" version = "2.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, - { name = "sympy" }, + { name = "filelock", marker = "sys_platform != 'linux'" }, + { name = "fsspec", marker = "sys_platform != 'linux'" }, + { name = "jinja2", marker = "sys_platform != 'linux'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform != 'linux') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'linux') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "setuptools", marker = "python_full_version >= '3.12' and sys_platform != 'linux'" }, + { name = "sympy", marker = "sys_platform != 'linux'" }, { name = "triton", marker = "sys_platform == 'never'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/bb/86/245c240d2138c17ed572c943c289056c2721abab70810d772c6bf5495b28/torch-2.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:030bbfe367379ae6a4ae4042b6c44da25383343b8b3c68abaa9c7231efbaf2dd", size = 104213554, upload-time = "2025-10-15T15:45:59.798Z" }, @@ -5830,8 +5478,8 @@ dependencies = [ { name = "docstring-parser" }, { name = "filelock" }, { name = "fsspec" }, - { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, + { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, { name = "pyre-extensions" }, { name = "pyyaml" }, { name = "tabulate" }, @@ -6342,32 +5990,20 @@ name = "wrapt" version = "2.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] sdist = { url = "https://files.pythonhosted.org/packages/49/19/5e5bcd855d808892fe02d49219f97a50f64cd6d8313d75df3494ee97b1a3/wrapt-2.0.0.tar.gz", hash = "sha256:35a542cc7a962331d0279735c30995b024e852cf40481e384fd63caaa391cbb9", size = 81722, upload-time = "2025-10-19T23:47:54.07Z" } wheels = [ @@ -6637,53 +6273,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, ] -[[package]] -name = "zarr" -version = "2.18.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "asciitree", marker = "python_full_version < '3.11'" }, - { name = "fasteners", marker = "python_full_version < '3.11' and sys_platform != 'emscripten'" }, - { name = "numcodecs", version = "0.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/c4/187a21ce7cf7c8f00c060dd0e04c2a81139bb7b1ab178bba83f2e1134ce2/zarr-2.18.3.tar.gz", hash = "sha256:2580d8cb6dd84621771a10d31c4d777dca8a27706a1a89b29f42d2d37e2df5ce", size = 3603224, upload-time = "2024-09-04T23:20:16.595Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/c9/142095e654c2b97133ff71df60979422717b29738b08bc8a1709a5d5e0d0/zarr-2.18.3-py3-none-any.whl", hash = "sha256:b1f7dfd2496f436745cdd4c7bcf8d3b4bc1dceef5fdd0d589c87130d842496dd", size = 210723, upload-time = "2024-09-04T23:20:14.491Z" }, -] - -[[package]] -name = "zarr" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "donfig", marker = "python_full_version >= '3.11'" }, - { name = "numcodecs", version = "0.16.3", source = { registry = "https://pypi.org/simple" }, extra = ["crc32c"], marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/67/14be68a7bad15eecda09b1e81fca2420f7533645fe187bf4d6104c1aad52/zarr-3.1.3.tar.gz", hash = "sha256:01342f3e26a02ed5670db608a5576fbdb8d76acb5c280bd2d0082454b1ba6f79", size = 349125, upload-time = "2025-09-18T19:32:41.688Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/71/9de7229515a53d1cc5705ca9c411530f711a2242f962214d9dbfe2741aa4/zarr-3.1.3-py3-none-any.whl", hash = "sha256:45f67f87f65f14fa453f99dd8110a5936b7ac69f3a21981d33e90407c80c302a", size = 276427, upload-time = "2025-09-18T19:32:40.042Z" }, -] - [[package]] name = "zipp" version = "3.23.0"