diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 088877304a7..4a838b24d95 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -48,7 +48,9 @@ inputs: is_ci_workload: description: "Is CI workload" required: true - + is_merge_group: + description: "Is merge group" + required: true runs: using: "composite" steps: @@ -179,7 +181,12 @@ runs: #!/bin/bash set -euxo pipefail - if [ "${{ steps.has-run-tests-label.outputs.main }}" == "true" ]; then + if [ "${{ inputs.is_merge_group }}" == "true" ]; then + ARGS=( + --scope mr-github + --n-repeat 1 + ) + elif [ "${{ steps.has-run-tests-label.outputs.main }}" == "true" ]; then ARGS=( --scope mr-github --enable-lightweight-mode diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index b27348f2dce..074f4234a91 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -18,8 +18,6 @@ on: - cron: 0 0 * * * push: branches: - - dev - - main - "pull-request/[0-9]+" - "deploy-release/*" merge_group: @@ -181,7 +179,7 @@ jobs: cicd-wait-in-queue: runs-on: ubuntu-latest needs: [pre-flight, linting] - environment: ${{ needs.pre-flight.outputs.is_merge_group == 'true' && 'merge-gate' || 'test' }} + environment: "test" if: | !(needs.pre-flight.outputs.is_ci_workload == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' @@ -404,7 +402,6 @@ jobs: success() || needs.pre-flight.outputs.is_ci_workload == 'true' || needs.pre-flight.outputs.force_run_all == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' ) && !cancelled() outputs: @@ -442,7 +439,7 @@ jobs: id: main env: HAS_RUN_TESTS_LABEL: ${{ steps.has-run-tests-label.outputs.main }} - HAS_RUN_FUNCTIONAL_TESTS_LABEL: ${{ steps.has-run-functional-tests-label.outputs.main }} + HAS_RUN_FUNCTIONAL_TESTS_LABEL: ${{ steps.has-run-functional-tests-label.outputs.main == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' }} run: | export PYTHONPATH=$(pwd) @@ -505,7 +502,6 @@ jobs: || needs.pre-flight.outputs.is_ci_workload == 'true' || needs.pre-flight.outputs.force_run_all == 'true' ) - && needs.pre-flight.outputs.is_merge_group == 'false' && !cancelled() steps: - name: Checkout @@ -521,6 +517,7 @@ jobs: PAT: ${{ secrets.PAT }} container-image: ${{ env.container-registry }}/megatron-lm:${{ github.sha }} is_ci_workload: ${{ needs.pre-flight.outputs.is_ci_workload }} + is_merge_group: ${{ needs.pre-flight.outputs.is_merge_group }} Nemo_CICD_Test: needs: diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index 90eddc55c27..a97a4d7bb38 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -110,14 +110,14 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: @@ -129,7 +129,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -142,28 +142,28 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # outdated TE: #501 - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #436 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -177,28 +177,28 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -212,7 +212,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -247,83 +247,83 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -331,14 +331,14 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -350,26 +350,26 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -383,28 +383,28 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -425,14 +425,14 @@ products: - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # Outdated: #502 @@ -469,7 +469,7 @@ products: - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -483,7 +483,7 @@ products: - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] diff --git a/tests/test_utils/recipes/mamba.yaml b/tests/test_utils/recipes/mamba.yaml index 47b731f7e00..456a6cbccf7 100644 --- a/tests/test_utils/recipes/mamba.yaml +++ b/tests/test_utils/recipes/mamba.yaml @@ -3,7 +3,7 @@ format_version: 1 maintainers: [mcore] loggers: [stdout] spec: - name: '{test_case}_{environment}_{platforms}' + name: "{test_case}_{environment}_{platforms}" model: hybrid build: mcore-pyt-{environment} nodes: 1 @@ -58,7 +58,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -74,7 +74,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp2_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml index 06039d77440..10c1140ddf5 100644 --- a/tests/test_utils/recipes/moe.yaml +++ b/tests/test_utils/recipes/moe.yaml @@ -119,7 +119,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] # products: @@ -139,17 +139,17 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4] products: @@ -161,12 +161,12 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: @@ -181,19 +181,19 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] scope: [nightly] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: