ci(fix): Run tests label (#1970)

.github/workflows/auto-assign-milestone.yml

@@ -14,6 +14,7 @@ jobs:
   assign-milestone:
     runs-on: ubuntu-latest
     environment: nemo-ci
+    if: github.repository == 'NVIDIA/Megatron-LM'
     steps:
       - name: Get PR info
         id: get-pr-info

.github/workflows/auto-reminder-bot.yml

@@ -0,0 +1,34 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+name: Auto Reminder Bot
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * *"
+
+jobs:
+  run-script:
+    environment: main
+    name: Run Auto Reminder Bot
+    runs-on: ubuntu-latest
+    if: github.repository == 'NVIDIA/Megatron-LM'
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir PyGithub slack-sdk
+
+      - name: Run Auto Reminder Bot
+        run: |
+          export SLACK_TOKEN=${{ secrets.SLACK_TOKEN }}
+          export SLACK_WEBHOOK_URL=${{ secrets.SLACK_WEBHOOK_URL }}
+          export GH_TOKEN=${{ secrets.PAT }}
+          python tests/test_utils/python_scripts/auto_reminder_github.py

.github/workflows/auto-swap-labels.yml

@@ -0,0 +1,33 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+name: Auto Swap Labels
+on:
+  pull_request_review:
+    types: [submitted]
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  check-approval:
+    runs-on: ubuntu-latest
+    if: github.event.review.state == 'approved' && github.repository == 'NVIDIA/Megatron-LM'
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir PyGithub slack-sdk
+
+      - name: Run Auto Reminder Bot
+        run: |
+          export GH_TOKEN=${{ github.token }}
+          export PR_NUMBER=${{ github.event.pull_request.number }}
+          python tests/test_utils/python_scripts/swap_pr_labels.py

.github/workflows/build-test-publish-wheel.yml

@@ -35,13 +35,15 @@ permissions:
 jobs:
   pre-flight:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5
+    if: github.repository == 'NVIDIA/Megatron-LM'
 
   build-test-publish-wheel:
     needs: [pre-flight]
     if: |
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_merge_group == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
+      && github.repository == 'NVIDIA/Megatron-LM'
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.63.1
     with:
       dry-run: true
@@ -68,6 +70,7 @@ jobs:
         || needs.pre-flight.outputs.is_deployment_workflow == 'true'
         || always()
       )
+      && github.repository == 'NVIDIA/Megatron-LM'
       && !cancelled()
     runs-on: ubuntu-latest
     steps:

.github/workflows/cherry-pick-release-commit.yml

@@ -22,6 +22,7 @@ on:
 jobs:
   cherry-pick:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.65.9
+    if: github.repository == 'NVIDIA/Megatron-LM'
     with:
       target-branches-pattern: 'core_(*dev_)?r[0-9]+\.[0-9]+\.[0-9]+'
     secrets:

.github/workflows/cicd-approve-test-queue.yml

@@ -23,6 +23,7 @@ jobs:
   approve-queue:
     runs-on: ubuntu-latest
     environment: main
+    if: github.repository == 'NVIDIA/Megatron-LM'
     strategy:
       matrix:
         branch: [main, dev]

.github/workflows/cicd-main.yml

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 name: CICD Megatron-LM
 on:
   schedule:
@@ -150,6 +151,7 @@ jobs:
 
   pre-flight:
     needs: [is-not-external-contributor]
+    if: github.repository == 'NVIDIA/Megatron-LM'
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.10
 
   linting:
@@ -251,11 +253,6 @@ jobs:
           apt-get update
           apt-get install -y gh
 
-      - name: Pull cache
-        run: |
-          docker pull ${{ env.container-registry }}/megatron-lm:main || true
-          docker pull ${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} || true
-
       - name: Get last merged PR
         id: cache_from
         env:
@@ -271,13 +268,16 @@ jobs:
                 }
               }
             }' | jq -r '.data.repository.pullRequests.nodes[].number' | while read -r number; do
-              echo "${{ env.container-registry }}/megatron-lm:$number"
+              echo "type=registry,ref=${{ env.container-registry }}/megatron-lm:$number-buildcache,mode=max"
             done)
 
           echo "LAST_PRS<<EOF" | tee -a $GITHUB_OUTPUT
           echo "$LAST_PRS" | tee -a $GITHUB_OUTPUT
           echo "EOF" | tee -a $GITHUB_OUTPUT
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
@@ -288,9 +288,11 @@ jobs:
           build-args: |
             FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:25.09-py3
           cache-from: |
-            ${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}
-            ${{ env.container-registry }}/megatron-lm:main
+            type=registry,ref=${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}-buildcache,mode=max
+            type=registry,ref=${{ env.container-registry }}/megatron-lm:main-buildcache,mode=max
             ${{ steps.cache_from.outputs.LAST_PRS }}
+          cache-to: |
+            type=registry,ref=${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}-buildcache,mode=max
           no-cache: false
           tags: |
             ${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }}
@@ -368,6 +370,7 @@ jobs:
       - cicd-wait-in-queue
       - cicd-container-build
       - cicd-unit-tests-latest
+    environment: nemo-ci
     if: |
       (
         success() 
@@ -399,21 +402,26 @@ jobs:
       - name: Parse functional tests
         id: main
         env:
-          HAS_RUN_TESTS_LABEL: ${{ steps.has-run-tests-label.outputs.HAS_RUN_TESTS_LABEL }}
+          HAS_RUN_TESTS_LABEL: ${{ steps.has-run-tests-label.outputs.main }}
         run: |
           export PYTHONPATH=$(pwd)
 
           if [ "$HAS_RUN_TESTS_LABEL" == "true" ]; then
             ARGS=(
-              --scope mr
+              --scope mr-github
               --enable-lightweight-mode
             )
+            echo ":warning: The Run tests label is not yet supported."
           else
             ARGS=(
               --scope mr-slim
             )
           fi
 
+          ARGS=(
+            --scope mr-slim
+          )
+
           python tests/test_utils/python_scripts/generate_jet_trigger_job.py \
             --n-repeat 5 \
             --time-limit 2700 \

.github/workflows/close-inactive-issue-pr.yml

@@ -18,4 +18,5 @@ on:
 
 jobs:
   close-issues:
+    if: github.repository == 'NVIDIA/Megatron-LM'
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_close_inactive_issue_pr.yml@v0.44.0

.github/workflows/community-bot.yml

@@ -22,6 +22,7 @@ on:
 jobs:
   community-bot:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_community_bot.yml@v0.65.10
+    if: github.repository == 'NVIDIA/Megatron-LM'
     secrets:
       GH_TOKEN: ${{ secrets.PAT }}
       environment: main

.github/workflows/copyright-check.yml

@@ -21,17 +21,21 @@ on:
       - main
       - "pull-request/[0-9]+"
       - "deploy-release/*"
+  merge_group:
+    types: [checks_requested]
 
 jobs:
   pre-flight:
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.64.2
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.10
+    if: github.repository == 'NVIDIA/Megatron-LM'
 
   copyright-check:
     needs: [pre-flight]
     if: |
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_copyright_check.yml@v0.65.11
+      && github.repository == 'NVIDIA/Megatron-LM'
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_copyright_check.yml@v0.65.12
 
   copyright-check-summary:
     needs: [pre-flight, copyright-check]
@@ -42,9 +46,12 @@ jobs:
         || always()
       )
       && !cancelled()
+      && github.repository == 'NVIDIA/Megatron-LM'
     runs-on: ubuntu-latest
     steps:
       - name: Result
+        env:
+          SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }}
         run: |
           FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
 

.github/workflows/dependabot.yml

@@ -14,6 +14,7 @@ jobs:
     environment: nemo-ci
     outputs:
       mcore: ${{ steps.get-branch.outputs.mcore_release_branch }}
+    if: github.repository == 'NVIDIA/Megatron-LM'
     steps:
       - name: Get release branch names
         id: get-branch
@@ -28,6 +29,7 @@ jobs:
 
   bump-tags:
     needs: [get-release-branch-names]
+    if: github.repository == 'NVIDIA/Megatron-LM'
     strategy:
       fail-fast: false
       matrix:
@@ -46,7 +48,7 @@ jobs:
       SSH_PWD: ${{ secrets.SSH_PWD }}
 
   notify:
-    if: failure()
+    if: failure() && github.repository == 'NVIDIA/Megatron-LM'
     runs-on: ubuntu-latest
     needs: [bump-tags]
     steps:

.github/workflows/install-test.yml

@@ -30,13 +30,15 @@ on:
 jobs:
   pre-flight:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5
+    if: github.repository == 'NVIDIA/Megatron-LM'
 
   pip-test-pytorch:
     needs: [pre-flight]
     if: |
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_merge_group == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
+      && github.repository == 'NVIDIA/Megatron-LM'
     runs-on: linux-amd64-cpu16
     name: Pip - Python${{ matrix.python-version }} - AMD64/Linux - NGC PyTorch
     container:
@@ -82,6 +84,7 @@ jobs:
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_merge_group == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
+      && github.repository == 'NVIDIA/Megatron-LM'
     runs-on: linux-amd64-cpu16
     name: UV - Python${{ matrix.python-version }} - AMD64/Linux - NGC PyTorch
     container:
@@ -135,6 +138,7 @@ jobs:
         || always()
       )
       && !cancelled()
+      && github.repository == 'NVIDIA/Megatron-LM'
     steps:
       - name: Get workflow result
         id: result

.gitlab/stages/05.publish.yml

@@ -784,7 +784,7 @@ publish:approve_merge_gate:
         export COMMENT="Main is healthy. Submitting PR."
       elif [[ $EXIT_CODE -eq 1 ]]; then
         export STATUS="rejected"
-        export COMMENT="Main is not healthy. An automation engineer is investigating. No need to take any action."
+        export COMMENT="$TARGET_BRANCH is not healthy. An automation engineer is investigating. No need to take any action."
       elif [[ $EXIT_CODE -eq 2 ]]; then
         echo "Main is running. We won't cancel the deployment."
         exit 0

tests/test_utils/python_scripts/launch_nemo_run_workload.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
 import logging
 import os
 import pathlib