From edc5d26e888e2b0fe737797aeda137f85824e53a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 8 Jul 2026 15:57:25 +0000 Subject: [PATCH] ci: revert unify legacy scope names (#5316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 68f4c6271a6302737fbcd7bd4bcae8150110797e (PR #5316). Restores the legacy functional-test scope vocabulary (`mr`, `mr-github`, `nightly`, ...) and the prior recipe_parser / trigger_internal_ci behavior. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: oliver könig --- .gitlab-ci.yml | 9 +- .gitlab/stages/04.functional-tests.yml | 52 ++-- .../python_scripts/recipe_parser.py | 64 ++--- tests/test_utils/recipes/gb200/gpt-1node.yaml | 92 +++---- .../recipes/gb200/gpt-perf-dp4.yaml | 2 +- tests/test_utils/recipes/gb200/gpt-perf.yaml | 2 +- tests/test_utils/recipes/gb200/gpt.yaml | 172 ++++-------- .../recipes/gb200/hybrid-perf-ep4.yaml | 2 +- .../test_utils/recipes/gb200/hybrid-perf.yaml | 2 +- tests/test_utils/recipes/gb200/moe-1node.yaml | 24 +- .../recipes/gb200/moe-dynamic-inference.yaml | 2 +- tests/test_utils/recipes/gb200/moe.yaml | 56 ++-- tests/test_utils/recipes/h100/bert.yaml | 24 +- tests/test_utils/recipes/h100/flextron.yaml | 3 +- ...pt-dynamic-inference-with-coordinator.yaml | 12 +- .../recipes/h100/gpt-dynamic-inference.yaml | 46 ++-- tests/test_utils/recipes/h100/gpt-grads.yaml | 2 +- tests/test_utils/recipes/h100/gpt-grpo.yaml | 6 +- .../h100/gpt-inference-server-smoke.yaml | 2 +- .../recipes/h100/gpt-offline-inference.yaml | 6 +- .../test_utils/recipes/h100/gpt-perf-dp8.yaml | 2 +- tests/test_utils/recipes/h100/gpt-perf.yaml | 2 +- .../recipes/h100/gpt-static-inference.yaml | 8 +- tests/test_utils/recipes/h100/gpt.yaml | 257 +++++++----------- .../recipes/h100/hybrid-perf-ep8.yaml | 2 +- .../test_utils/recipes/h100/hybrid-perf.yaml | 2 +- .../recipes/h100/mamba-dynamic-inference.yaml | 8 +- .../recipes/h100/mamba-static-inference.yaml | 2 +- tests/test_utils/recipes/h100/mamba.yaml | 10 +- .../recipes/h100/module_performance.yaml | 2 +- ...oe-dynamic-inference-with-coordinator.yaml | 4 +- .../recipes/h100/moe-dynamic-inference.yaml | 8 +- tests/test_utils/recipes/h100/moe-grpo.yaml | 2 +- .../recipes/h100/moe-static-inference.yaml | 6 +- tests/test_utils/recipes/h100/moe.yaml | 68 ++--- .../recipes/h100/multimodal-llava.yaml | 4 +- tests/test_utils/recipes/h100/t5.yaml | 36 +-- tools/trigger_internal_ci.md | 4 +- tools/trigger_internal_ci.py | 24 +- 39 files changed, 410 insertions(+), 621 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6860865510..2eb1b43be0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ workflow: UNIT_TEST: "no" INTEGRATION_TEST: "no" FUNCTIONAL_TEST: "yes" - FUNCTIONAL_TEST_SCOPE: L1,L2 + FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 5 FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" FUNCTIONAL_TEST_TIME_LIMIT: 3600 @@ -126,7 +126,7 @@ workflow: UNIT_TEST_TIMEOUT: 30 INTEGRATION_TEST: "no" FUNCTIONAL_TEST: "yes" - FUNCTIONAL_TEST_SCOPE: L1,L2 + FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 CLUSTER_A100: "" @@ -204,10 +204,9 @@ variables: - "no" description: To run the funtional test suite FUNCTIONAL_TEST_SCOPE: - value: "L1,L2" + value: "mr" options: - - "L1,L2" - - "mr-slim" + - "mr" - "nightly" - "weekly" - "pre-release" diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index c023584bf38..515aa3e7f7f 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -66,28 +66,18 @@ functional:configure: RELEASE_ARGS=() fi - | - # NOTE: $FUNCTIONAL_TEST_SCOPE is supplied by workflow rules, scheduled - # pipelines, or manual dispatch. Legacy GitLab values (`mr-slim`, `mr`, - # `nightly`, `weekly`) and L-tier names (`L0`–`L4`) are accepted; comma- - # separated tiers union, e.g. `L1,L2` for the full MR functional suite. - # - # Recipe rows use the L-tier vocabulary; recipe_parser.py resolves filter - # input (tests/test_utils/python_scripts/recipe_parser.py): - # - `mr-github-slim` -> `L0` - # - `mr-github` -> `L1` - # - `mr` -> `L2` - # - `nightly` -> `L3`; `weekly` -> `L4` - # `unit-tests` and `release` are not aliased and match recipe rows verbatim. - # - # For the full MR suite (`L1,L2`), GB200 keeps the legacy GitLab `mr` - # footprint (`L2` only) rather than unioning L1 rows that were never in the - # old GB200 MR matrix. - SCOPE_A100_H100="$FUNCTIONAL_TEST_SCOPE" - SCOPE_GB200="$FUNCTIONAL_TEST_SCOPE" - if [[ "$FUNCTIONAL_TEST_SCOPE" == "L1,L2" ]]; then - SCOPE_GB200="L2" - fi - COMMON_ARGS=( + # NOTE: $FUNCTIONAL_TEST_SCOPE is supplied by external scheduled-pipeline + # configurations and uses the GitLab-side legacy values: `mr`, `mr-slim`, + # `nightly`, `weekly`, `unit-tests`, `release`. + # - `mr` / `mr-slim` / `unit-tests` / `release` pass through verbatim + # and match recipe rows tagged with the same string. They are + # intentionally NOT aliased (see LEGACY_SCOPE_ALIASES in + # tests/test_utils/python_scripts/recipe_parser.py) so the GitLab + # and GitHub matrices stay decoupled. + # - `nightly` / `weekly` are aliased to L2 / L3 by the parser and also + # inject a default cadence when the recipe has no explicit one. + ARGS=( + "--scope $FUNCTIONAL_TEST_SCOPE" "--n-repeat $FUNCTIONAL_TEST_REPEAT" "--time-limit $FUNCTIONAL_TEST_TIME_LIMIT" "--test-cases $FUNCTIONAL_TEST_CASES" @@ -131,8 +121,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_A100_H100" \ + ${ARGS[@]} \ --environment dev \ --platform dgx_a100 \ --cluster $A100_CLUSTER \ @@ -141,8 +130,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_A100_H100" \ + ${ARGS[@]} \ --environment dev \ --platform dgx_h100 \ --cluster $H100_CLUSTER \ @@ -151,8 +139,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_A100_H100" \ + ${ARGS[@]} \ --environment lts \ --platform dgx_a100 \ --cluster $A100_CLUSTER \ @@ -161,8 +148,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_A100_H100" \ + ${ARGS[@]} \ --environment lts \ --platform dgx_h100 \ --cluster $H100_CLUSTER \ @@ -171,8 +157,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_GB200" \ + ${ARGS[@]} \ --environment dev \ --platform dgx_gb200 \ --cluster $GB200_CLUSTER \ @@ -181,8 +166,7 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${COMMON_ARGS[@]} \ - --scope "$SCOPE_GB200" \ + ${ARGS[@]} \ --environment lts \ --platform dgx_gb200 \ --cluster $GB200_CLUSTER \ diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py index 9687b17c50d..dbebd2a3623 100644 --- a/tests/test_utils/python_scripts/recipe_parser.py +++ b/tests/test_utils/python_scripts/recipe_parser.py @@ -3,7 +3,7 @@ import itertools import logging import pathlib -from typing import List, Optional, Set +from typing import List, Optional import click import yaml @@ -20,53 +20,42 @@ # trigger, a default cadence. The tier acts purely as a suite/cost label; # cadence remains the trigger axis. # -# All legacy `scope` names (both GitHub-side `mr-github*` and GitLab-side -# `mr*`) are aliased onto the unified L-tier vocabulary so legacy callers and -# scheduled pipelines keep working: a `--scope mr-github` filter and a recipe -# row tagged `scope: [L1]` match the same tier. A literal `mr` tag or filter -# means a GitLab-only extra and resolves to the single tier `L2`. The slim PR -# sets (`mr-slim`, `mr-github-slim`) both map to `L0`. +# Only GitHub-side scopes (`mr-github-slim`, `mr-github`) are aliased onto the +# L-tier names. GitLab-only scopes (`mr`, `mr-slim`, `unit-tests`) are +# intentionally left as pass-through so GitLab `--scope mr*` / `--scope +# unit-tests` continue to match recipes verbatim and don't bleed into the +# GitHub L0 / L1 matrix. # # L-tier vocabulary (cost class, ascending): `L0` (slim PR) < `L1` (full PR / -# merge queue) < `L2` (GitLab MR extra) < `L3` (nightly) < `L4` (weekly). -# `L0-smoke` is a sub-L0 tier for fast lightweight smoke tests; it passes -# through verbatim and GitLab selects it via `--scope L0-smoke`. +# merge queue) < `L2` (nightly) < `L3` (weekly). `L0-smoke` is a sub-L0 tier +# for fast lightweight smoke tests (cheaper than `L0`); it is GitLab-only and +# passes through verbatim (not aliased here). Recipes tag rows with +# `scope: [L0-smoke]` and GitLab selects them via `--scope L0-smoke`. LEGACY_SCOPE_ALIASES = { - "mr-slim": ("L0", None), + # GitHub-only scopes are aliased onto the L-tier vocabulary so the GH CI + # workflow can filter on `L0` / `L1`. GitLab-only scopes (`mr`, `mr-slim`) + # are intentionally NOT aliased: they pass through to recipe rows verbatim + # and remain matchable by GitLab's `--scope mr-slim` / `--scope mr` calls, + # without bleeding into the GitHub `L0` / `L1` matrix. "mr-github-slim": ("L0", None), "mr-github": ("L1", None), - "mr": ("L2", None), - "nightly": ("L3", ["nightly"]), - "weekly": ("L4", ["weekly"]), + "nightly": ("L2", ["nightly"]), + "weekly": ("L3", ["weekly"]), } def _resolve_scope_alias(scope_value: str) -> str: """Resolve a legacy scope value to its L-tier alias (or return it unchanged). - Applied when flattening recipe rows. For `--scope` filters, use - `_resolve_scope_filter` so comma-separated values (e.g. `L1,L2`) work. + Applied both to recipe rows when flattening and to the `--scope` filter + input, so callers can pass either the legacy name (e.g. `nightly`) or the + new L-tier name (e.g. `L2`) and hit the same recipe rows. """ if scope_value in LEGACY_SCOPE_ALIASES: return LEGACY_SCOPE_ALIASES[scope_value][0] return scope_value -def _resolve_scope_filter(scope: str) -> Set[str]: - """Resolve a `--scope` filter to the set of L-tier names to match. - - Supports comma-separated tiers, e.g. ``--scope L1,L2`` for a union. Each - token is legacy-aliased to a single tier. - """ - tiers: Set[str] = set() - for part in scope.split(","): - part = part.strip() - if not part: - continue - tiers.add(_resolve_scope_alias(part)) - return tiers - - def _apply_scope_alias(scope_value: str, explicit_cadence: Optional[List[str]]) -> tuple: """Resolve a legacy scope value to (new_scope, cadence). @@ -243,17 +232,18 @@ def filter_by_test_case(workload_manifests: List[dotdict], test_case: str) -> Op def filter_by_scope(workload_manifests: List[dotdict], scope: str) -> List[dotdict]: - """Returns all workloads whose scope is in the resolved filter tier set. + """Returns all workload with matching scope. - The filter input is legacy-aliased per token. Pass comma-separated tiers for a - union, e.g. ``--scope L1,L2`` (full GitLab MR) or legacy names such as - ``--scope mr-github``. + The filter input is run through the same legacy-scope alias as recipe + rows, so callers passing the legacy name (e.g. `--scope nightly`, + `--scope mr-github`) match recipes that have already been rewritten to + the new L-tier vocabulary (e.g. `scope: [L2]`, `scope: [L1]`). """ - resolved_scopes = _resolve_scope_filter(scope) + resolved_scope = _resolve_scope_alias(scope) workload_manifests = list( workload_manifest for workload_manifest in workload_manifests - if workload_manifest.spec["scope"] in resolved_scopes + if workload_manifest.spec["scope"] == resolved_scope ) if len(workload_manifests) == 0: diff --git a/tests/test_utils/recipes/gb200/gpt-1node.yaml b/tests/test_utils/recipes/gb200/gpt-1node.yaml index 66f26ce719b..9b009735963 100644 --- a/tests/test_utils/recipes/gb200/gpt-1node.yaml +++ b/tests/test_utils/recipes/gb200/gpt-1node.yaml @@ -70,27 +70,27 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [L0, L1] + scope: [mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_reruns_persistent_1_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node] products: - environment: [dev] - scope: [L0, L1] + scope: [mr-github, mr-github-slim] platforms: [dgx_gb200] ####################################################################### # tp1_pp1 variants # @@ -98,17 +98,17 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp1_pp2 variants # @@ -116,12 +116,12 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] ####################################################################### # tp1_pp4 variants # @@ -129,57 +129,57 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp1 variants # @@ -187,22 +187,22 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp2 (no cp) variants # @@ -210,42 +210,42 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_mla_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp2_cp2 variants (PP reduced 2→1 for 1-node) # @@ -253,47 +253,47 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_cp2_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] ####################################################################### # tp4_pp1 variants # @@ -301,17 +301,17 @@ products: - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp4_pp2 variant (PP reduced 2→1 for 1-node) # @@ -319,5 +319,5 @@ products: - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml index c650752b104..fb02a6f167d 100644 --- a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml +++ b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml @@ -43,5 +43,5 @@ products: - test_case: [gpt_583m_perf_gb200_4gpu] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt-perf.yaml b/tests/test_utils/recipes/gb200/gpt-perf.yaml index e4a2b439b45..9bbd6cf33b7 100644 --- a/tests/test_utils/recipes/gb200/gpt-perf.yaml +++ b/tests/test_utils/recipes/gb200/gpt-perf.yaml @@ -45,5 +45,5 @@ products: - test_case: [gpt_16b_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt.yaml b/tests/test_utils/recipes/gb200/gpt.yaml index 4208af2425f..a7ea92e8b99 100644 --- a/tests/test_utils/recipes/gb200/gpt.yaml +++ b/tests/test_utils/recipes/gb200/gpt.yaml @@ -73,45 +73,35 @@ products: - test_case: [gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp2] products: - environment: [dev] - scope: [L3] - cadence: [nightly] - platforms: [dgx_gb200] - - environment: [dev] - scope: [L0-smoke] + scope: [nightly, L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp2_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp1_pp4] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp4_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp4_pp1_resume_torch] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] ####################################################################### # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # @@ -120,28 +110,22 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [L3] - cadence: [nightly] - platforms: [dgx_gb200] - - environment: [dev] - scope: [L0-smoke] + scope: [nightly, L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: @@ -150,56 +134,47 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] # products: @@ -209,181 +184,152 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_gdn] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp1_pp4_memory_speed] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp4_pp1_memory_speed] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] ####################################################################### # Super important mr, mr-github tests that run for DEV per mr, mr-github # @@ -391,39 +337,35 @@ products: - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_weekly_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] products: - environment: [dev] - scope: [L4] - cadence: [weekly] + scope: [weekly] platforms: [dgx_gb200] - test_case: [gpt3_weekly_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] products: - environment: [dev] - scope: [L4] - cadence: [weekly] + scope: [weekly] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml index d8f01ae20cc..f34b66dae5c 100644 --- a/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml +++ b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml @@ -43,5 +43,5 @@ products: - test_case: [hybrid_nanov3_3b_perf_gb200_4gpu] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf.yaml b/tests/test_utils/recipes/gb200/hybrid-perf.yaml index 2c7ac486558..78585557c42 100644 --- a/tests/test_utils/recipes/gb200/hybrid-perf.yaml +++ b/tests/test_utils/recipes/gb200/hybrid-perf.yaml @@ -44,5 +44,5 @@ products: - test_case: [hybrid_2b_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe-1node.yaml b/tests/test_utils/recipes/gb200/moe-1node.yaml index 8b9523c1689..54a674b9a6f 100644 --- a/tests/test_utils/recipes/gb200/moe-1node.yaml +++ b/tests/test_utils/recipes/gb200/moe-1node.yaml @@ -64,22 +64,22 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2_1node] products: - environment: [dev] - scope: [L0, L1] + scope: [mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node] products: - environment: [dev] - scope: [L0, L1] + scope: [mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] ####################################################################### # tp1_pp2 MoE variants # @@ -87,7 +87,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp1 MoE variants # @@ -95,27 +95,27 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node] products: - environment: [dev] - scope: [L1] + scope: [mr-github] platforms: [dgx_gb200] ####################################################################### # ep8 variant (EP reduced 8→4 for 1-node) # @@ -123,7 +123,7 @@ products: - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp4_ep2_etp2_pp2 variant (PP reduced 2→1 for 1-node) # @@ -131,5 +131,5 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph_1node] products: - environment: [dev] - scope: [L1-temp-disabled] + scope: [mr-github-temp-disabled] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml b/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml index 7ebbeede2c9..d1d6ea865b4 100644 --- a/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml @@ -61,5 +61,5 @@ products: - test_case: [gpt_dynamic_inference_tp2_pp2_ep2_gptoss_20b_swa] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe.yaml b/tests/test_utils/recipes/gb200/moe.yaml index 3e2004901bd..8f934ae13ab 100644 --- a/tests/test_utils/recipes/gb200/moe.yaml +++ b/tests/test_utils/recipes/gb200/moe.yaml @@ -67,47 +67,37 @@ products: - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] - platforms: [dgx_gb200] - - environment: [dev] - scope: [L0-smoke] + scope: [nightly, L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] # products: # non-determinism: #478 @@ -123,8 +113,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] # products: @@ -144,26 +133,22 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -183,30 +168,27 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: - environment: [dev] - scope: [L0, L2] + scope: [mr, mr-slim] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] products: - environment: [dev] - scope: [L0, L2] + scope: [mr, mr-slim] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: @@ -232,7 +214,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: @@ -243,5 +225,5 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/h100/bert.yaml b/tests/test_utils/recipes/h100/bert.yaml index e307f5dba70..0f638a83151 100644 --- a/tests/test_utils/recipes/h100/bert.yaml +++ b/tests/test_utils/recipes/h100/bert.yaml @@ -61,45 +61,35 @@ products: - test_case: [bert_mcore_tp2_pp2] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_local_spec] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_resume_torch_dist_local_spec] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp1_pp2] products: - environment: [dev] - scope: [L3] - cadence: [nightly] - platforms: [dgx_h100] - - environment: [dev] - scope: [L0-smoke] + scope: [nightly, L0-smoke] platforms: [dgx_h100] - test_case: [bert_mcore_tp1_pp4_vp2] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp4_pp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/flextron.yaml b/tests/test_utils/recipes/h100/flextron.yaml index 9f04fb4df76..4b89f78037e 100644 --- a/tests/test_utils/recipes/h100/flextron.yaml +++ b/tests/test_utils/recipes/h100/flextron.yaml @@ -60,6 +60,5 @@ products: - test_case: [hybrid_flextron_nightly_tp2_pp1_ep2_dgx_h100_1N8G] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml index ed00906a603..fdc96221e44 100644 --- a/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml @@ -60,32 +60,32 @@ products: - test_case: [gpt_dynamic_inference_tp8_pp1_dp1_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp8_dp1_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_dp2_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_prefix_caching_longest_prefix_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_prefix_caching_round_robin_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] # Throughput coverage for this 583M dynamic-batching DP=8 config now lives at # tests/performance_tests/test_cases/gpt/gpt_583m_perf/ — see diff --git a/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml index 244ee6025fb..43661c16cd3 100644 --- a/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml @@ -65,116 +65,116 @@ products: - test_case: [gpt_dynamic_inference_tp8_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_flashinfer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_chunked_prefill] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_cuda_graphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_chunked_prefill_cuda_graphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] # ===== Round 2 (Tier 2) - parallelism x feature, 3-way combos, sampling, memory variants ===== - test_case: [gpt_dynamic_inference_tp2_pp2_583m_prefix_caching] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp8_pp1_583m_prefix_caching] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp8_583m_prefix_caching] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_chunked_prefill] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_583m_flashinfer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_cuda_graphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill_cuda_graphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_prefix_caching_cuda_graphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill_flashinfer] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_top_p_sampling] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_stop_words] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_top_n_logprobs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_lru] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_uvm_level1] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-grads.yaml b/tests/test_utils/recipes/h100/gpt-grads.yaml index d2f20d20966..c8ff49c47f6 100644 --- a/tests/test_utils/recipes/h100/gpt-grads.yaml +++ b/tests/test_utils/recipes/h100/gpt-grads.yaml @@ -64,5 +64,5 @@ products: - test_case: [gpt3_mcore_reruns_resume_check_grads] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-grpo.yaml b/tests/test_utils/recipes/h100/gpt-grpo.yaml index ddae6dfdca2..b2c2ce5bb99 100644 --- a/tests/test_utils/recipes/h100/gpt-grpo.yaml +++ b/tests/test_utils/recipes/h100/gpt-grpo.yaml @@ -59,17 +59,17 @@ products: - test_case: [gpt_grpo_basic_function] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_throughput] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_throughput_github] products: diff --git a/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml b/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml index 3baa92ef0c8..7002f28d8c6 100644 --- a/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml +++ b/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml @@ -45,5 +45,5 @@ products: - test_case: [gpt_inference_server_smoke_tp1_pp1_dp8_583m] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-offline-inference.yaml b/tests/test_utils/recipes/h100/gpt-offline-inference.yaml index 744e2bdb141..8451993b19f 100644 --- a/tests/test_utils/recipes/h100/gpt-offline-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-offline-inference.yaml @@ -58,15 +58,15 @@ products: - test_case: [gpt_offline_inference_sync_tp1_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_offline_inference_sync_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_offline_inference_async_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml b/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml index 249b1588982..6d9cc4e948f 100644 --- a/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml +++ b/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml @@ -41,5 +41,5 @@ products: - test_case: [gpt_583m_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-perf.yaml b/tests/test_utils/recipes/h100/gpt-perf.yaml index 3b9df4ab125..88817a9b2df 100644 --- a/tests/test_utils/recipes/h100/gpt-perf.yaml +++ b/tests/test_utils/recipes/h100/gpt-perf.yaml @@ -41,5 +41,5 @@ products: - test_case: [gpt_16b_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-static-inference.yaml b/tests/test_utils/recipes/h100/gpt-static-inference.yaml index 1ef698708d2..c904a533bfa 100644 --- a/tests/test_utils/recipes/h100/gpt-static-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-static-inference.yaml @@ -60,20 +60,20 @@ products: - test_case: [gpt_static_inference_tp1_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_583m_cudagraphs] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_583m_fp8_cudagraphs] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_16b_multiprompt_tokensmatch] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt.yaml b/tests/test_utils/recipes/h100/gpt.yaml index e8365638b3c..3e88e325048 100644 --- a/tests/test_utils/recipes/h100/gpt.yaml +++ b/tests/test_utils/recipes/h100/gpt.yaml @@ -65,57 +65,45 @@ products: - test_case: [gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp1_pp2] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp1_pp2_resume_torch_dist] products: - environment: [dev, lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp1_pp4] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp1_pp4_resume_torch_dist] products: - environment: [dev, lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp4_pp1_resume_torch] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp4_pp1_resume_torch_dist] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] ####################################################################### # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # @@ -124,32 +112,29 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [L1, L2, L0-smoke] + scope: [mr, mr-github, L0-smoke] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L2, mr-github-broken] + scope: [mr, mr-github-broken] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: # - environment: [dev] @@ -159,72 +144,66 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # outdated TE: #501 - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #436 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #437 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] # products: # - environment: [dev] @@ -235,271 +214,247 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: # - environment: [dev] # scope: [mr] # platforms: [dgx_h100] # Hangs: #513 - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_gdn] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_sync] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_async] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_async_mcore] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_dsa] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_7b_tp1_pp4_memory_speed] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # OOM: #434 - test_case: [gpt3_7b_tp4_pp1_memory_speed] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # OOM: #434 - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # Outdated: #502 @@ -515,7 +470,7 @@ products: - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [L2, mr-github-broken] + scope: [mr, mr-github-broken] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] @@ -529,46 +484,40 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_weekly_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] products: - environment: [dev] - scope: [L4] - cadence: [weekly] + scope: [weekly] platforms: [dgx_h100] - test_case: [gpt3_weekly_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] products: - environment: [dev] - scope: [L4] - cadence: [weekly] + scope: [weekly] platforms: [dgx_h100] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_uninstall_te] # products: diff --git a/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml b/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml index 6223fc2e32b..5c81e98b49d 100644 --- a/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml +++ b/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml @@ -40,5 +40,5 @@ products: - test_case: [hybrid_nanov3_3b_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/hybrid-perf.yaml b/tests/test_utils/recipes/h100/hybrid-perf.yaml index 56b4f0abcf8..11835061ec2 100644 --- a/tests/test_utils/recipes/h100/hybrid-perf.yaml +++ b/tests/test_utils/recipes/h100/hybrid-perf.yaml @@ -40,5 +40,5 @@ products: - test_case: [hybrid_2b_perf] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml b/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml index f72a1595f8a..56f56c17a82 100644 --- a/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml @@ -60,20 +60,20 @@ products: - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_chunked_prefill] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_flashinfer] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_mamba_bf16_states] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/mamba-static-inference.yaml b/tests/test_utils/recipes/h100/mamba-static-inference.yaml index b9e42668d9f..bafabf084fc 100644 --- a/tests/test_utils/recipes/h100/mamba-static-inference.yaml +++ b/tests/test_utils/recipes/h100/mamba-static-inference.yaml @@ -65,5 +65,5 @@ products: - test_case: [hybrid_static_inference_tp1_pp1_2B_cudagraphs] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dg x_h100] diff --git a/tests/test_utils/recipes/h100/mamba.yaml b/tests/test_utils/recipes/h100/mamba.yaml index 0ec42cc8a22..77331a751e4 100644 --- a/tests/test_utils/recipes/h100/mamba.yaml +++ b/tests/test_utils/recipes/h100/mamba.yaml @@ -60,7 +60,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -68,7 +68,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp4_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -76,7 +76,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp2_vpp2_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -84,7 +84,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp2_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -92,7 +92,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp2_pp1_cp4_dgx_a100_1N8G] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] diff --git a/tests/test_utils/recipes/h100/module_performance.yaml b/tests/test_utils/recipes/h100/module_performance.yaml index 577907f2362..96e109498da 100644 --- a/tests/test_utils/recipes/h100/module_performance.yaml +++ b/tests/test_utils/recipes/h100/module_performance.yaml @@ -50,5 +50,5 @@ products: - test_case: [moe_perf] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index 9934a470d14..1ecc7247c72 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -60,12 +60,12 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_zmq] products: - environment: [dev] - scope: [L2] #[mr, mr-github] - broken on github after NGC PyTorch26.02 + scope: [mr] #[mr, mr-github] - broken on github after NGC PyTorch26.02 platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [L2] #[mr, mr-github] - broken on github after NGC PyTorch26.02 + scope: [mr] #[mr, mr-github] - broken on github after NGC PyTorch26.02 platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_zmq_suspend_resume] products: diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml index 57feb6cd6da..889542638e4 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml @@ -65,20 +65,20 @@ products: - test_case: [gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_ep2_gptoss_20b_swa] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_ep4_16B_prefix_caching] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_ep4_16B_chunked_prefill] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-grpo.yaml b/tests/test_utils/recipes/h100/moe-grpo.yaml index 040d7e15047..fcad7111f05 100644 --- a/tests/test_utils/recipes/h100/moe-grpo.yaml +++ b/tests/test_utils/recipes/h100/moe-grpo.yaml @@ -59,5 +59,5 @@ products: - test_case: [gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-static-inference.yaml b/tests/test_utils/recipes/h100/moe-static-inference.yaml index e3c5ac2bf48..e82b452b2b9 100644 --- a/tests/test_utils/recipes/h100/moe-static-inference.yaml +++ b/tests/test_utils/recipes/h100/moe-static-inference.yaml @@ -60,15 +60,15 @@ products: - test_case: [gpt_static_inference_tp1_pp1_ep1_16B_logitsmatch] products: - environment: [dev] - scope: [L2, mr-github-broken] + scope: [mr, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [L2, mr-github-broken] + scope: [mr, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt_static_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [L2, mr-github-broken] + scope: [mr, mr-github-broken] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml index 145005c87ce..69b02ce639f 100644 --- a/tests/test_utils/recipes/h100/moe.yaml +++ b/tests/test_utils/recipes/h100/moe.yaml @@ -65,60 +65,48 @@ products: - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] # products: # non-determinism: #478 # - environment: [dev, lts] @@ -133,7 +121,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] # products: @@ -153,22 +141,22 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp1_pp1_te_4experts_groupedGEMM_op_fuser] products: - environment: [dev] - scope: [L1, L2, L0-smoke] + scope: [mr, mr-github, L0-smoke] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4] products: @@ -176,12 +164,11 @@ products: # scope: [mr] # platforms: [dgx_h100] # hang: #513 - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -191,30 +178,29 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [L3] - cadence: [nightly] + scope: [nightly] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: @@ -229,7 +215,7 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2_ep_overlap] products: - environment: [dev] - scope: [L1, L2] + scope: [mr, mr-github] platforms: [dgx_h100] ####################################################################### # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # @@ -250,7 +236,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [L0, L1, L2] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: diff --git a/tests/test_utils/recipes/h100/multimodal-llava.yaml b/tests/test_utils/recipes/h100/multimodal-llava.yaml index 42337ac1206..0c6ff97a002 100644 --- a/tests/test_utils/recipes/h100/multimodal-llava.yaml +++ b/tests/test_utils/recipes/h100/multimodal-llava.yaml @@ -63,10 +63,10 @@ products: - test_case: [multimodal_llava_mcore_te_tp1_pp1] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] - test_case: [multimodal_llava_mcore_te_tp4_sp_cp2] products: - environment: [dev] - scope: [L2] + scope: [mr] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/t5.yaml b/tests/test_utils/recipes/h100/t5.yaml index 917dec3dc3d..039a7beafb6 100644 --- a/tests/test_utils/recipes/h100/t5.yaml +++ b/tests/test_utils/recipes/h100/t5.yaml @@ -61,69 +61,55 @@ products: - test_case: [t5_11b_mcore_tp4_pp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp4_pp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_tp4_pp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp1_pp1_vp1_resume_torch] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_te_tp2_pp1_vp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_te_tp2_pp1_vp1_sequence_parallel] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp1_pp1_vp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] - platforms: [dgx_a100, dgx_h100] - - environment: [dev] - scope: [L0-smoke] + scope: [nightly, L0-smoke] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp1_pp1_vp1_resume_torch] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp2_pp1_vp1] products: - environment: [dev] - scope: [L3] - cadence: [nightly] + scope: [nightly] platforms: [dgx_a100, dgx_h100] diff --git a/tools/trigger_internal_ci.md b/tools/trigger_internal_ci.md index fb6ce63906c..8d3a8577537 100644 --- a/tools/trigger_internal_ci.md +++ b/tools/trigger_internal_ci.md @@ -37,7 +37,7 @@ python -m pip install python-gitlab python tools/trigger_internal_ci.py \ --gitlab-origin gitlab \ [--access-token glpat-] \ - [--functional-test-scope L1,L2] \ + [--functional-test-scope mr] \ [--functional-test-repeat 5] \ [--functional-test-cases all] \ [--functional-test-name release-testing/mcore-vX.Y.Z] \ @@ -49,7 +49,7 @@ python tools/trigger_internal_ci.py \ |---|---|---| | `--gitlab-origin` | *(required)* | Git remote name for the internal GitLab | | `--access-token` | `$GITLAB_TOKEN` | Personal access token with `api` scope | -| `--functional-test-scope` | `L1,L2` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | +| `--functional-test-scope` | `mr` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | | `--functional-test-repeat` | `5` | `FUNCTIONAL_TEST_REPEAT` pipeline variable | | `--functional-test-cases` | `all` | `FUNCTIONAL_TEST_CASES` pipeline variable | | `--functional-test-name` | commit SHA | `FUNCTIONAL_TEST_NAME` pipeline variable — names the run for `pre-release`/`release` scopes (used as the run name and W&B experiment). | diff --git a/tools/trigger_internal_ci.py b/tools/trigger_internal_ci.py index 33b87cb3713..d46a2f6436c 100644 --- a/tools/trigger_internal_ci.py +++ b/tools/trigger_internal_ci.py @@ -40,10 +40,6 @@ "INTEGRATION_TEST": "no", } -# No trigger-side scope aliases are applied. The internal GitLab CI / recipe -# parser resolves legacy scope values server-side; bare `mr` maps to `L2`. -SCOPE_ALIASES = {} - # Scopes whose recipes run full convergence/checkpointing workloads and need a # long wall-clock budget. The default short-scope time limit is left untouched. LONG_RUNNING_SCOPES = ("release", "weekly") @@ -52,11 +48,6 @@ logger = logging.getLogger(__name__) -def normalize_scope(scope): - """Map a legacy FUNCTIONAL_TEST_SCOPE value to its L-tier equivalent.""" - return SCOPE_ALIASES.get(scope, scope) - - def resolve_time_limit(scope, override): """Resolve the FUNCTIONAL_TEST_TIME_LIMIT value for a functional test scope. @@ -159,9 +150,8 @@ def main(): ) parser.add_argument( "--functional-test-scope", - default="L1,L2", - help="FUNCTIONAL_TEST_SCOPE pipeline variable (default: L1,L2). " - "Legacy 'mr' resolves server-side to L2.", + default="mr", + help="FUNCTIONAL_TEST_SCOPE pipeline variable (default: mr)", ) parser.add_argument( "--functional-test-repeat", @@ -230,17 +220,9 @@ def main(): git_push(args.gitlab_origin, target_branch, dry_run=args.dry_run) - functional_test_scope = normalize_scope(args.functional_test_scope) - if functional_test_scope != args.functional_test_scope: - logger.info( - "Normalized FUNCTIONAL_TEST_SCOPE '%s' -> '%s'", - args.functional_test_scope, - functional_test_scope, - ) - pipeline_vars = { **PIPELINE_VARIABLES_FIXED, - "FUNCTIONAL_TEST_SCOPE": functional_test_scope, + "FUNCTIONAL_TEST_SCOPE": args.functional_test_scope, "FUNCTIONAL_TEST_REPEAT": str(args.functional_test_repeat), "FUNCTIONAL_TEST_CASES": args.functional_test_cases, }