diff --git a/.github/workflows/auto-assign-milestone.yml b/.github/workflows/auto-assign-milestone.yml index 7eae6838332..8153728f9fd 100644 --- a/.github/workflows/auto-assign-milestone.yml +++ b/.github/workflows/auto-assign-milestone.yml @@ -14,6 +14,7 @@ jobs: assign-milestone: runs-on: ubuntu-latest environment: nemo-ci + if: github.repository == 'NVIDIA/Megatron-LM' steps: - name: Get PR info id: get-pr-info diff --git a/.github/workflows/auto-reminder-bot.yml b/.github/workflows/auto-reminder-bot.yml new file mode 100644 index 00000000000..c3aa8169b50 --- /dev/null +++ b/.github/workflows/auto-reminder-bot.yml @@ -0,0 +1,34 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +name: Auto Reminder Bot + +on: + workflow_dispatch: + schedule: + - cron: "0 12 * * *" + +jobs: + run-script: + environment: main + name: Run Auto Reminder Bot + runs-on: ubuntu-latest + if: github.repository == 'NVIDIA/Megatron-LM' + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + pip install --no-cache-dir PyGithub slack-sdk + + - name: Run Auto Reminder Bot + run: | + export SLACK_TOKEN=${{ secrets.SLACK_TOKEN }} + export SLACK_WEBHOOK_URL=${{ secrets.SLACK_WEBHOOK_URL }} + export GH_TOKEN=${{ secrets.PAT }} + python tests/test_utils/python_scripts/auto_reminder_github.py diff --git a/.github/workflows/auto-swap-labels.yml b/.github/workflows/auto-swap-labels.yml new file mode 100644 index 00000000000..5335026e2af --- /dev/null +++ b/.github/workflows/auto-swap-labels.yml @@ -0,0 +1,33 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +name: Auto Swap Labels +on: + pull_request_review: + types: [submitted] + +permissions: + pull-requests: write + contents: read + +jobs: + check-approval: + runs-on: ubuntu-latest + if: github.event.review.state == 'approved' && github.repository == 'NVIDIA/Megatron-LM' + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + pip install --no-cache-dir PyGithub slack-sdk + + - name: Run Auto Reminder Bot + run: | + export GH_TOKEN=${{ github.token }} + export PR_NUMBER=${{ github.event.pull_request.number }} + python tests/test_utils/python_scripts/swap_pr_labels.py diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml index 1ff9f53202b..0f3a037979a 100644 --- a/.github/workflows/build-test-publish-wheel.yml +++ b/.github/workflows/build-test-publish-wheel.yml @@ -35,6 +35,7 @@ permissions: jobs: pre-flight: uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5 + if: github.repository == 'NVIDIA/Megatron-LM' build-test-publish-wheel: needs: [pre-flight] @@ -42,6 +43,7 @@ jobs: !(needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true') + && github.repository == 'NVIDIA/Megatron-LM' uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.63.1 with: dry-run: true @@ -68,6 +70,7 @@ jobs: || needs.pre-flight.outputs.is_deployment_workflow == 'true' || always() ) + && github.repository == 'NVIDIA/Megatron-LM' && !cancelled() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/cherry-pick-release-commit.yml b/.github/workflows/cherry-pick-release-commit.yml index 9cf8ed98660..58b447939a7 100644 --- a/.github/workflows/cherry-pick-release-commit.yml +++ b/.github/workflows/cherry-pick-release-commit.yml @@ -22,6 +22,7 @@ on: jobs: cherry-pick: uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.65.9 + if: github.repository == 'NVIDIA/Megatron-LM' with: target-branches-pattern: 'core_(*dev_)?r[0-9]+\.[0-9]+\.[0-9]+' secrets: diff --git a/.github/workflows/cicd-approve-test-queue.yml b/.github/workflows/cicd-approve-test-queue.yml index 1f23905d5d8..ccc8327368d 100644 --- a/.github/workflows/cicd-approve-test-queue.yml +++ b/.github/workflows/cicd-approve-test-queue.yml @@ -23,6 +23,7 @@ jobs: approve-queue: runs-on: ubuntu-latest environment: main + if: github.repository == 'NVIDIA/Megatron-LM' strategy: matrix: branch: [main, dev] diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index d1e411be98f..27e1f6cdacb 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + name: CICD Megatron-LM on: schedule: @@ -150,6 +151,7 @@ jobs: pre-flight: needs: [is-not-external-contributor] + if: github.repository == 'NVIDIA/Megatron-LM' uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.10 linting: @@ -251,11 +253,6 @@ jobs: apt-get update apt-get install -y gh - - name: Pull cache - run: | - docker pull ${{ env.container-registry }}/megatron-lm:main || true - docker pull ${{ env.container-registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} || true - - name: Get last merged PR id: cache_from env: @@ -271,13 +268,16 @@ jobs: } } }' | jq -r '.data.repository.pullRequests.nodes[].number' | while read -r number; do - echo "${{ env.container-registry }}/megatron-lm:$number" + echo "type=registry,ref=${{ env.container-registry }}/megatron-lm:$number-buildcache,mode=max" done) echo "LAST_PRS< latest_reviews[review.user.login].submitted_at + ): + latest_reviews[review.user.login] = review + except Exception as e: + logger.warning(f"Could not get reviews for PR #{pr.number}: {e}") + + # 2. Separate reviewers into approvers (List B) and non-approvers + approvers = {user for user, review in latest_reviews.items() if review.state == "APPROVED"} + non_approving_reviewers = { + user for user, review in latest_reviews.items() if review.state == "CHANGES_REQUESTED" + } + + # 3. Get all *currently pending* review requests + try: + pending_users_req, pending_teams_req = pr.get_review_requests() + pending_individuals = {r.login for r in pending_users_req} + pending_teams_slugs = {t.slug for t in pending_teams_req} + except Exception as e: + logger.warning(f"Could not get review requests for PR #{pr.number}: {e}") + pending_individuals = set() + pending_teams_slugs = set() + + # 4. Filter pending teams based on the current stage + teams_to_query = ( + pending_teams_slugs - self.EXCLUDED_TEAMS + if self.stage == self.EXPERT_REVIEW + else pending_teams_slugs & self.EXCLUDED_TEAMS + ) + + # 5. Get members from the required pending teams + pending_team_members = set() + for slug in teams_to_query: + try: + pending_team_members.update( + m.login for m in self.org.get_team_by_slug(slug).get_members() + ) + except Exception as e: + logger.warning(f"Could not get members for team {slug} on PR #{pr.number}: {e}") + + # 6. "List A": Combine all users who *still need to review* + all_required_reviewers = ( + pending_individuals | pending_team_members | non_approving_reviewers + ) + + # 7. Final list (List A - List B): + pending_reviewers = all_required_reviewers - approvers + logger.info(f"Pending reviewers: {pending_reviewers}") + if len(pending_reviewers) == 0: + try: + pr.remove_from_labels(self.EXPERT_REVIEW) + logger.info(f'Removed "{self.EXPERT_REVIEW}" label from PR #{pr.number}') + except Exception as e: + logger.warning( + f'Failed to remove "{self.EXPERT_REVIEW}" label from PR #{pr.number}: {e}' + ) + + try: + pr.add_to_labels(self.FINAL_REVIEW) + logger.info(f'Added "{self.FINAL_REVIEW}" label to PR #{pr.number}') + except Exception as e: + logger.warning(f'Failed to add "{self.FINAL_REVIEW}" label to PR #{pr.number}: {e}') + + +def main(): + token = os.environ.get("GH_TOKEN") + repo = os.environ.get("REPO", "NVIDIA/Megatron-LM") + pr_number = int(os.environ.get("PR_NUMBER")) + + if not token: + logger.error("GH_TOKEN environment variable is required") + sys.exit(1) + + logger.info(f"Starting PR review reminder for {repo}") + tracker = PRReviewTracker(token, repo, pr_number) + tracker.swap_labels() + + +if __name__ == "__main__": + main() diff --git a/tests/test_utils/recipes/ckpt_converter.yaml b/tests/test_utils/recipes/ckpt_converter.yaml index f78f184a326..bf328ae44c9 100644 --- a/tests/test_utils/recipes/ckpt_converter.yaml +++ b/tests/test_utils/recipes/ckpt_converter.yaml @@ -48,7 +48,7 @@ products: - test_case: [ckpt_converter] products: - environment: [dev] - scope: [mr-broken] + scope: [mr-github-broken, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly-broken] diff --git a/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml b/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml index 47b8d346150..f4a7d6c786b 100644 --- a/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml +++ b/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml @@ -47,5 +47,5 @@ products: - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_validation] products: - environment: [dev] - scope: [mr-broken] + scope: [mr-broken, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/gpt-dynamic-inference.yaml index 748e4734a6d..77a98d4bd7f 100644 --- a/tests/test_utils/recipes/gpt-dynamic-inference.yaml +++ b/tests/test_utils/recipes/gpt-dynamic-inference.yaml @@ -72,5 +72,5 @@ products: - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/gpt-grads.yaml b/tests/test_utils/recipes/gpt-grads.yaml index cdd3a050ff2..bf048542410 100644 --- a/tests/test_utils/recipes/gpt-grads.yaml +++ b/tests/test_utils/recipes/gpt-grads.yaml @@ -62,5 +62,5 @@ products: - test_case: [gpt3_mcore_reruns_resume_check_grads] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index 0dafb8685c2..baf07cb9759 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -104,75 +104,75 @@ products: scope: [nightly] platforms: [dgx_h100] ####################################################################### - # MR tests: Mostly DEV on MR, and LTS on nightly cadence, except for # + # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # # some very important tests. # ####################################################################### - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # - environment: [lts] # scope: [nightly] # Non-deterministic: #487 - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # outdated TE: #501 - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #436 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #437 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -193,42 +193,42 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # Hangs: #513 # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # Hangs: #513 - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied] products: # - environment: [dev] - # scope: [mr] # Hangs: #513 + # scope: [mr, mr-github] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap] products: # - environment: [dev] - # scope: [mr] # Hangs: #513 + # scope: [mr, mr-github] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -326,14 +326,14 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -345,49 +345,49 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader] products: # - environment: [dev] - # scope: [mr] # Hangs: #513 + # scope: [mr, mr-github] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -415,25 +415,25 @@ products: - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # Outdated: #502 # - test_case: [gpt3_mcore_te_tp2_pp1_fsdp2_resume_torch_dist] # products: # - environment: [dev] - # scope: [mr] # Broken: #484 + # scope: [mr, mr-github] # Broken: #484 # - environment: [lts] # scope: [nightly] # Requires PyT 2.4: #481 ####################################################################### - # Super important MR tests that run for both DEV and LTS per MR # + # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # ####################################################################### - test_case: [gpt3_mcore_reruns_persistent_1] products: @@ -445,19 +445,16 @@ products: # - test_case: [gpt3_mcore_reruns_persistent_2] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [lts] - scope: [mr] - - environment: [dev] scope: [mr, mr-github] - platforms: [dgx_h100] - environment: [dev] - scope: [mr-slim] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: @@ -465,43 +462,40 @@ products: scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [mr] + scope: [mr, mr-github] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [lts] - scope: [mr] - - environment: [dev] scope: [mr, mr-github] - platforms: [dgx_h100] - environment: [dev] - scope: [mr-slim] + scope: [mr, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [mr] + scope: [mr, mr-github] # - test_case: [gpt3_mcore_te_tp4_pp2_frozen_resume_torch_dist_reshard_8x1xNone] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp1_pp1_frozen_resume_torch_dist_dist_optimizer] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp1_pp4_frozen_resume_torch_dist_swiglu] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_cp2_nondeterministic] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_a100, dgx_h100] # - test_case: [gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] # products: @@ -551,4 +545,4 @@ products: # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_uninstall_te] # products: # - environment: [dev, lts] - # scope: [mr] # Non-deterministic: #483 + # scope: [mr, mr-github] # Non-deterministic: #483 diff --git a/tests/test_utils/recipes/mamba-static-inference.yaml b/tests/test_utils/recipes/mamba-static-inference.yaml index e727c4db5ee..9fcc86830f0 100644 --- a/tests/test_utils/recipes/mamba-static-inference.yaml +++ b/tests/test_utils/recipes/mamba-static-inference.yaml @@ -62,5 +62,5 @@ products: - test_case: [hybrid_static_inference_tp1_pp1_2B_cudagraphs] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dg x_h100] diff --git a/tests/test_utils/recipes/mamba.yaml b/tests/test_utils/recipes/mamba.yaml index 0f8a4085ea5..40d1d095aa4 100644 --- a/tests/test_utils/recipes/mamba.yaml +++ b/tests/test_utils/recipes/mamba.yaml @@ -67,7 +67,7 @@ products: # - test_case: [hybrid_mr_mcore_te_tp1_pp4_cp1_dgx_a100_1N8G] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] diff --git a/tests/test_utils/recipes/moe-dynamic-inference.yaml b/tests/test_utils/recipes/moe-dynamic-inference.yaml index c9d1be57add..d477bdeda4a 100644 --- a/tests/test_utils/recipes/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/moe-dynamic-inference.yaml @@ -62,5 +62,5 @@ products: - test_case: [gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/moe-static-inference.yaml b/tests/test_utils/recipes/moe-static-inference.yaml index c11cd294592..bd7c4ca0f50 100644 --- a/tests/test_utils/recipes/moe-static-inference.yaml +++ b/tests/test_utils/recipes/moe-static-inference.yaml @@ -57,15 +57,15 @@ products: - test_case: [gpt_static_inference_tp1_pp1_ep1_16B_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_static_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml index 8164ca37df8..893ab40981b 100644 --- a/tests/test_utils/recipes/moe.yaml +++ b/tests/test_utils/recipes/moe.yaml @@ -78,28 +78,28 @@ products: # Weekly tests: Run both DEV and LTS unless something is flaky # ####################################################################### ####################################################################### - # MR tests: Mostly DEV on MR, and LTS on nightly cadence, except for # + # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # # some very important tests. # ####################################################################### - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # hang: #513 # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # hang: #513 - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: @@ -122,20 +122,20 @@ products: - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] ####################################################################### - # Super important MR tests that run for both DEV and LTS per MR # + # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # ####################################################################### # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_te_8experts2parallel_dist_optimizer] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_te_8experts2parallel_groupedGEMM] # products: # - environment: [dev] - # scope: [mr] + # scope: [mr, mr-github] # platforms: [dgx_h100] ########################### # Merge train tests # @@ -143,18 +143,12 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] - platforms: [dgx_h100] - - environment: [dev] - scope: [mr-slim] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - environment: [dev] - scope: [mr-slim] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] products: