From fa0af7f0b5ae310382e29cc3b45fb3fde42060ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 10:35:19 +0000 Subject: [PATCH 1/8] ci: Debug integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .github/workflows/cicd-main.yml | 9 ++------- tests/functional_tests/shell_test_utils/_run_training.sh | 2 -- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c7fad39bbbd..fb120d947b8 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -368,7 +368,7 @@ jobs: - pre-flight - cicd-wait-in-queue - cicd-container-build - - cicd-unit-tests-latest + # - cicd-unit-tests-latest environment: nemo-ci if: | ( @@ -410,17 +410,12 @@ jobs: --scope mr-github --enable-lightweight-mode ) - echo ":warning: The Run tests label is not yet supported." else ARGS=( --scope mr-slim ) fi - ARGS=( - --scope mr-slim - ) - python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ --n-repeat 5 \ --time-limit 2700 \ @@ -451,7 +446,7 @@ jobs: - pre-flight - cicd-wait-in-queue - cicd-parse-integration-tests - - cicd-unit-tests-latest + # - cicd-unit-tests-latest runs-on: nvidia-ci-aws-gpu-x8 name: "${{ matrix.model }}/${{ matrix.test_case }} - latest" environment: nemo-ci diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh index 1d0e77a3477..2ce1e56571d 100644 --- a/tests/functional_tests/shell_test_utils/_run_training.sh +++ b/tests/functional_tests/shell_test_utils/_run_training.sh @@ -170,8 +170,6 @@ DISTRIBUTED_ARGS=( --master_port $MASTER_PORT --node_rank $NODE_RANK --log-dir $LOG_DIR - --tee "0:3,7:3" - --redirects "3" ) # Start training From a161dabbf3f48771e4f5beb68186efe58b91ceb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 10:42:35 +0000 Subject: [PATCH 2/8] cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- hello_world | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hello_world diff --git a/hello_world b/hello_world deleted file mode 100644 index e69de29bb2d..00000000000 From ed6f34f085af6a7908dee830f45b854071384739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 11:02:38 +0000 Subject: [PATCH 3/8] fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- tests/test_utils/recipes/ckpt_converter.yaml | 2 +- .../recipes/gpt-dynamic-inference-cuda-graphs.yaml | 2 +- tests/test_utils/recipes/unit-tests.yaml | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_utils/recipes/ckpt_converter.yaml b/tests/test_utils/recipes/ckpt_converter.yaml index bf328ae44c9..ea62ccbdee1 100644 --- a/tests/test_utils/recipes/ckpt_converter.yaml +++ b/tests/test_utils/recipes/ckpt_converter.yaml @@ -48,7 +48,7 @@ products: - test_case: [ckpt_converter] products: - environment: [dev] - scope: [mr-github-broken, mr-github] + scope: [mr-broken, mr-github-broken] platforms: [dgx_h100] - environment: [lts] scope: [nightly-broken] diff --git a/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml b/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml index f4a7d6c786b..5f7ae5c2b03 100644 --- a/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml +++ b/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml @@ -47,5 +47,5 @@ products: - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_validation] products: - environment: [dev] - scope: [mr-broken, mr-github] + scope: [mr-broken, mr-github-broken] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/unit-tests.yaml b/tests/test_utils/recipes/unit-tests.yaml index 5d2243a94af..8e9421ac02e 100644 --- a/tests/test_utils/recipes/unit-tests.yaml +++ b/tests/test_utils/recipes/unit-tests.yaml @@ -59,6 +59,13 @@ spec: cp coverage.xml {assets_dir} products: + - test_case: [tests/unit_tests/test_model_configs.py] + products: + - environment: [lts, dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] - test_case: [tests/unit_tests/test_fp8_param.py] products: - environment: [lts, dev] From f2929a8bddaadb9a948a3dddafc561dec7d39790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 11:31:23 +0000 Subject: [PATCH 4/8] update workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .../workflows/build-test-publish-wheel.yml | 74 ++++++++++++++++--- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml index 327765d3f5a..52db2ed5760 100644 --- a/.github/workflows/build-test-publish-wheel.yml +++ b/.github/workflows/build-test-publish-wheel.yml @@ -36,6 +36,71 @@ jobs: uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5 if: github.repository == 'NVIDIA/Megatron-LM' + build-wheel: + strategy: + matrix: + include: + - PACKAGE: megatron-core + PLATFORM: arm64 + IMAGE: quay.io/pypa/manylinux_2_28_aarch64 + - PACKAGE: megatron-core + PLATFORM: amd64 + IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + - PACKAGE: megatron-fsdp + IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + PLATFORM: amd64 + runs-on: ubuntu-latest + container: ${{ matrix.IMAGE }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build wheel + run: | + PUBLISH_DRYRUN=yes + + if [ "$PACKAGE" = "megatron-core" ]; then + ROOTDIR="megatron/core" + BUILD_DIR="." + elif [ "$PACKAGE" = "megatron-fsdp" ]; then + ROOTDIR="megatron/core/distributed/fsdp/src/megatron_fsdp" + BUILD_DIR="megatron/core/distributed/fsdp/src" + else + echo Unknown package: $PACKAGE + exit 1 + fi + + if [ "$PUBLISH_DRYRUN" = "yes" ]; then + PRE_RELEASE=$(sed -n "s/.*PRE_RELEASE = '\(.*\)'/\1/p" $ROOTDIR/package_info.py) + sed -i "/^PRE_RELEASE/c\PRE_RELEASE = '${PRE_RELEASE}.dev$((RANDOM % 900000 + 100000))'" $ROOTDIR/package_info.py + fi + + pushd $BUILD_DIR + /opt/python/cp310-cp310/bin/python -m build + /opt/python/cp311-cp311/bin/python -m build + /opt/python/cp312-cp312/bin/python -m build + /opt/python/cp313-cp313/bin/python -m build + USE_DIST_DIR=0 + auditwheel repair dist/*.whl || USE_DIST_DIR=1 + if [ "$USE_DIST_DIR" != "1" ]; then + rm -rf dist/*.whl + fi + popd + + pushd $ROOTDIR + EXPECTED_RELEASE_NUMBER=$(/opt/python/cp312-cp312/bin/python -c "import package_info; print(package_info.__version__)") + popd + + echo "EXPECTED_RELEASE_NUMBER_${PLATFORM}_${PACKAGE//-/_}=$EXPECTED_RELEASE_NUMBER" | tee -a "${GITHUB_OUTPUT}" + mkdir -p wheelhouse/ + + if [ "$PACKAGE" = "megatron-fsdp" ]; then + cp -a megatron/core/distributed/fsdp/src/dist/* wheelhouse/ + fi + + ls -al wheelhouse/ || true + ls -al dist/ || true + build-test-publish-wheel: needs: [pre-flight] if: | @@ -44,15 +109,6 @@ jobs: || needs.pre-flight.outputs.is_deployment_workflow == 'true') && github.repository == 'NVIDIA/Megatron-LM' uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.63.1 - with: - dry-run: true - python-package: megatron.core - python-version: "3.10" - packaging: uv - no-publish: ${{ !(github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) }} - custom-container: nvcr.io/nvidia/pytorch:25.05-py3 - no-build-isolation: true - runner: linux-amd64-cpu16 secrets: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} From 69eaa6789e5c5cb9291bf6bed14af6f450a94361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 11:31:31 +0000 Subject: [PATCH 5/8] flaky tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- tests/test_utils/recipes/gpt-dynamic-inference.yaml | 2 +- tests/test_utils/recipes/gpt.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils/recipes/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/gpt-dynamic-inference.yaml index 77a98d4bd7f..ad6a4a211e9 100644 --- a/tests/test_utils/recipes/gpt-dynamic-inference.yaml +++ b/tests/test_utils/recipes/gpt-dynamic-inference.yaml @@ -57,7 +57,7 @@ products: - test_case: [gpt_dynamic_inference_tp1_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr, mr-github-broken] # this test takes too long for github CI platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp8_pp1_583m_logitsmatch] products: diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index baf07cb9759..64da2a865e3 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -117,7 +117,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr, mr-github-broken] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -438,7 +438,7 @@ products: - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr, mr-github-broken] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] From ed5313a9199f43e0907581df0d82e558a2e085fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 11:53:47 +0000 Subject: [PATCH 6/8] test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .github/workflows/cicd-main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index fb120d947b8..c069abf370a 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -368,7 +368,7 @@ jobs: - pre-flight - cicd-wait-in-queue - cicd-container-build - # - cicd-unit-tests-latest + - cicd-unit-tests-latest environment: nemo-ci if: | ( @@ -446,7 +446,7 @@ jobs: - pre-flight - cicd-wait-in-queue - cicd-parse-integration-tests - # - cicd-unit-tests-latest + - cicd-unit-tests-latest runs-on: nvidia-ci-aws-gpu-x8 name: "${{ matrix.model }}/${{ matrix.test_case }} - latest" environment: nemo-ci From c017dd838da79b62f0d308885fd69eb37bb973af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 13:52:21 +0000 Subject: [PATCH 7/8] revert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .../workflows/build-test-publish-wheel.yml | 74 +++---------------- 1 file changed, 9 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml index 52db2ed5760..327765d3f5a 100644 --- a/.github/workflows/build-test-publish-wheel.yml +++ b/.github/workflows/build-test-publish-wheel.yml @@ -36,71 +36,6 @@ jobs: uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5 if: github.repository == 'NVIDIA/Megatron-LM' - build-wheel: - strategy: - matrix: - include: - - PACKAGE: megatron-core - PLATFORM: arm64 - IMAGE: quay.io/pypa/manylinux_2_28_aarch64 - - PACKAGE: megatron-core - PLATFORM: amd64 - IMAGE: quay.io/pypa/manylinux_2_28_x86_64 - - PACKAGE: megatron-fsdp - IMAGE: quay.io/pypa/manylinux_2_28_x86_64 - PLATFORM: amd64 - runs-on: ubuntu-latest - container: ${{ matrix.IMAGE }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Build wheel - run: | - PUBLISH_DRYRUN=yes - - if [ "$PACKAGE" = "megatron-core" ]; then - ROOTDIR="megatron/core" - BUILD_DIR="." - elif [ "$PACKAGE" = "megatron-fsdp" ]; then - ROOTDIR="megatron/core/distributed/fsdp/src/megatron_fsdp" - BUILD_DIR="megatron/core/distributed/fsdp/src" - else - echo Unknown package: $PACKAGE - exit 1 - fi - - if [ "$PUBLISH_DRYRUN" = "yes" ]; then - PRE_RELEASE=$(sed -n "s/.*PRE_RELEASE = '\(.*\)'/\1/p" $ROOTDIR/package_info.py) - sed -i "/^PRE_RELEASE/c\PRE_RELEASE = '${PRE_RELEASE}.dev$((RANDOM % 900000 + 100000))'" $ROOTDIR/package_info.py - fi - - pushd $BUILD_DIR - /opt/python/cp310-cp310/bin/python -m build - /opt/python/cp311-cp311/bin/python -m build - /opt/python/cp312-cp312/bin/python -m build - /opt/python/cp313-cp313/bin/python -m build - USE_DIST_DIR=0 - auditwheel repair dist/*.whl || USE_DIST_DIR=1 - if [ "$USE_DIST_DIR" != "1" ]; then - rm -rf dist/*.whl - fi - popd - - pushd $ROOTDIR - EXPECTED_RELEASE_NUMBER=$(/opt/python/cp312-cp312/bin/python -c "import package_info; print(package_info.__version__)") - popd - - echo "EXPECTED_RELEASE_NUMBER_${PLATFORM}_${PACKAGE//-/_}=$EXPECTED_RELEASE_NUMBER" | tee -a "${GITHUB_OUTPUT}" - mkdir -p wheelhouse/ - - if [ "$PACKAGE" = "megatron-fsdp" ]; then - cp -a megatron/core/distributed/fsdp/src/dist/* wheelhouse/ - fi - - ls -al wheelhouse/ || true - ls -al dist/ || true - build-test-publish-wheel: needs: [pre-flight] if: | @@ -109,6 +44,15 @@ jobs: || needs.pre-flight.outputs.is_deployment_workflow == 'true') && github.repository == 'NVIDIA/Megatron-LM' uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.63.1 + with: + dry-run: true + python-package: megatron.core + python-version: "3.10" + packaging: uv + no-publish: ${{ !(github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) }} + custom-container: nvcr.io/nvidia/pytorch:25.05-py3 + no-build-isolation: true + runner: linux-amd64-cpu16 secrets: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} From 8ce87d8080e3b709e44d32a63c7b4f5e8ded0dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 29 Oct 2025 14:23:49 +0000 Subject: [PATCH 8/8] revert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- tests/functional_tests/shell_test_utils/_run_training.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh index 2ce1e56571d..1d0e77a3477 100644 --- a/tests/functional_tests/shell_test_utils/_run_training.sh +++ b/tests/functional_tests/shell_test_utils/_run_training.sh @@ -170,6 +170,8 @@ DISTRIBUTED_ARGS=( --master_port $MASTER_PORT --node_rank $NODE_RANK --log-dir $LOG_DIR + --tee "0:3,7:3" + --redirects "3" ) # Start training