diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2eb1b43be0c..a6860865510 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ workflow: UNIT_TEST: "no" INTEGRATION_TEST: "no" FUNCTIONAL_TEST: "yes" - FUNCTIONAL_TEST_SCOPE: mr + FUNCTIONAL_TEST_SCOPE: L1,L2 FUNCTIONAL_TEST_REPEAT: 5 FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" FUNCTIONAL_TEST_TIME_LIMIT: 3600 @@ -126,7 +126,7 @@ workflow: UNIT_TEST_TIMEOUT: 30 INTEGRATION_TEST: "no" FUNCTIONAL_TEST: "yes" - FUNCTIONAL_TEST_SCOPE: mr + FUNCTIONAL_TEST_SCOPE: L1,L2 FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 CLUSTER_A100: "" @@ -204,9 +204,10 @@ variables: - "no" description: To run the funtional test suite FUNCTIONAL_TEST_SCOPE: - value: "mr" + value: "L1,L2" options: - - "mr" + - "L1,L2" + - "mr-slim" - "nightly" - "weekly" - "pre-release" diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index 515aa3e7f7f..c023584bf38 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -66,18 +66,28 @@ functional:configure: RELEASE_ARGS=() fi - | - # NOTE: $FUNCTIONAL_TEST_SCOPE is supplied by external scheduled-pipeline - # configurations and uses the GitLab-side legacy values: `mr`, `mr-slim`, - # `nightly`, `weekly`, `unit-tests`, `release`. - # - `mr` / `mr-slim` / `unit-tests` / `release` pass through verbatim - # and match recipe rows tagged with the same string. They are - # intentionally NOT aliased (see LEGACY_SCOPE_ALIASES in - # tests/test_utils/python_scripts/recipe_parser.py) so the GitLab - # and GitHub matrices stay decoupled. - # - `nightly` / `weekly` are aliased to L2 / L3 by the parser and also - # inject a default cadence when the recipe has no explicit one. - ARGS=( - "--scope $FUNCTIONAL_TEST_SCOPE" + # NOTE: $FUNCTIONAL_TEST_SCOPE is supplied by workflow rules, scheduled + # pipelines, or manual dispatch. Legacy GitLab values (`mr-slim`, `mr`, + # `nightly`, `weekly`) and L-tier names (`L0`–`L4`) are accepted; comma- + # separated tiers union, e.g. `L1,L2` for the full MR functional suite. + # + # Recipe rows use the L-tier vocabulary; recipe_parser.py resolves filter + # input (tests/test_utils/python_scripts/recipe_parser.py): + # - `mr-github-slim` -> `L0` + # - `mr-github` -> `L1` + # - `mr` -> `L2` + # - `nightly` -> `L3`; `weekly` -> `L4` + # `unit-tests` and `release` are not aliased and match recipe rows verbatim. + # + # For the full MR suite (`L1,L2`), GB200 keeps the legacy GitLab `mr` + # footprint (`L2` only) rather than unioning L1 rows that were never in the + # old GB200 MR matrix. + SCOPE_A100_H100="$FUNCTIONAL_TEST_SCOPE" + SCOPE_GB200="$FUNCTIONAL_TEST_SCOPE" + if [[ "$FUNCTIONAL_TEST_SCOPE" == "L1,L2" ]]; then + SCOPE_GB200="L2" + fi + COMMON_ARGS=( "--n-repeat $FUNCTIONAL_TEST_REPEAT" "--time-limit $FUNCTIONAL_TEST_TIME_LIMIT" "--test-cases $FUNCTIONAL_TEST_CASES" @@ -121,7 +131,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_A100_H100" \ --environment dev \ --platform dgx_a100 \ --cluster $A100_CLUSTER \ @@ -130,7 +141,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_A100_H100" \ --environment dev \ --platform dgx_h100 \ --cluster $H100_CLUSTER \ @@ -139,7 +151,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_A100_H100" \ --environment lts \ --platform dgx_a100 \ --cluster $A100_CLUSTER \ @@ -148,7 +161,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_A100_H100" \ --environment lts \ --platform dgx_h100 \ --cluster $H100_CLUSTER \ @@ -157,7 +171,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_GB200" \ --environment dev \ --platform dgx_gb200 \ --cluster $GB200_CLUSTER \ @@ -166,7 +181,8 @@ functional:configure: - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ - ${ARGS[@]} \ + ${COMMON_ARGS[@]} \ + --scope "$SCOPE_GB200" \ --environment lts \ --platform dgx_gb200 \ --cluster $GB200_CLUSTER \ diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py index dbebd2a3623..9687b17c50d 100644 --- a/tests/test_utils/python_scripts/recipe_parser.py +++ b/tests/test_utils/python_scripts/recipe_parser.py @@ -3,7 +3,7 @@ import itertools import logging import pathlib -from typing import List, Optional +from typing import List, Optional, Set import click import yaml @@ -20,42 +20,53 @@ # trigger, a default cadence. The tier acts purely as a suite/cost label; # cadence remains the trigger axis. # -# Only GitHub-side scopes (`mr-github-slim`, `mr-github`) are aliased onto the -# L-tier names. GitLab-only scopes (`mr`, `mr-slim`, `unit-tests`) are -# intentionally left as pass-through so GitLab `--scope mr*` / `--scope -# unit-tests` continue to match recipes verbatim and don't bleed into the -# GitHub L0 / L1 matrix. +# All legacy `scope` names (both GitHub-side `mr-github*` and GitLab-side +# `mr*`) are aliased onto the unified L-tier vocabulary so legacy callers and +# scheduled pipelines keep working: a `--scope mr-github` filter and a recipe +# row tagged `scope: [L1]` match the same tier. A literal `mr` tag or filter +# means a GitLab-only extra and resolves to the single tier `L2`. The slim PR +# sets (`mr-slim`, `mr-github-slim`) both map to `L0`. # # L-tier vocabulary (cost class, ascending): `L0` (slim PR) < `L1` (full PR / -# merge queue) < `L2` (nightly) < `L3` (weekly). `L0-smoke` is a sub-L0 tier -# for fast lightweight smoke tests (cheaper than `L0`); it is GitLab-only and -# passes through verbatim (not aliased here). Recipes tag rows with -# `scope: [L0-smoke]` and GitLab selects them via `--scope L0-smoke`. +# merge queue) < `L2` (GitLab MR extra) < `L3` (nightly) < `L4` (weekly). +# `L0-smoke` is a sub-L0 tier for fast lightweight smoke tests; it passes +# through verbatim and GitLab selects it via `--scope L0-smoke`. LEGACY_SCOPE_ALIASES = { - # GitHub-only scopes are aliased onto the L-tier vocabulary so the GH CI - # workflow can filter on `L0` / `L1`. GitLab-only scopes (`mr`, `mr-slim`) - # are intentionally NOT aliased: they pass through to recipe rows verbatim - # and remain matchable by GitLab's `--scope mr-slim` / `--scope mr` calls, - # without bleeding into the GitHub `L0` / `L1` matrix. + "mr-slim": ("L0", None), "mr-github-slim": ("L0", None), "mr-github": ("L1", None), - "nightly": ("L2", ["nightly"]), - "weekly": ("L3", ["weekly"]), + "mr": ("L2", None), + "nightly": ("L3", ["nightly"]), + "weekly": ("L4", ["weekly"]), } def _resolve_scope_alias(scope_value: str) -> str: """Resolve a legacy scope value to its L-tier alias (or return it unchanged). - Applied both to recipe rows when flattening and to the `--scope` filter - input, so callers can pass either the legacy name (e.g. `nightly`) or the - new L-tier name (e.g. `L2`) and hit the same recipe rows. + Applied when flattening recipe rows. For `--scope` filters, use + `_resolve_scope_filter` so comma-separated values (e.g. `L1,L2`) work. """ if scope_value in LEGACY_SCOPE_ALIASES: return LEGACY_SCOPE_ALIASES[scope_value][0] return scope_value +def _resolve_scope_filter(scope: str) -> Set[str]: + """Resolve a `--scope` filter to the set of L-tier names to match. + + Supports comma-separated tiers, e.g. ``--scope L1,L2`` for a union. Each + token is legacy-aliased to a single tier. + """ + tiers: Set[str] = set() + for part in scope.split(","): + part = part.strip() + if not part: + continue + tiers.add(_resolve_scope_alias(part)) + return tiers + + def _apply_scope_alias(scope_value: str, explicit_cadence: Optional[List[str]]) -> tuple: """Resolve a legacy scope value to (new_scope, cadence). @@ -232,18 +243,17 @@ def filter_by_test_case(workload_manifests: List[dotdict], test_case: str) -> Op def filter_by_scope(workload_manifests: List[dotdict], scope: str) -> List[dotdict]: - """Returns all workload with matching scope. + """Returns all workloads whose scope is in the resolved filter tier set. - The filter input is run through the same legacy-scope alias as recipe - rows, so callers passing the legacy name (e.g. `--scope nightly`, - `--scope mr-github`) match recipes that have already been rewritten to - the new L-tier vocabulary (e.g. `scope: [L2]`, `scope: [L1]`). + The filter input is legacy-aliased per token. Pass comma-separated tiers for a + union, e.g. ``--scope L1,L2`` (full GitLab MR) or legacy names such as + ``--scope mr-github``. """ - resolved_scope = _resolve_scope_alias(scope) + resolved_scopes = _resolve_scope_filter(scope) workload_manifests = list( workload_manifest for workload_manifest in workload_manifests - if workload_manifest.spec["scope"] == resolved_scope + if workload_manifest.spec["scope"] in resolved_scopes ) if len(workload_manifests) == 0: diff --git a/tests/test_utils/recipes/gb200/gpt-1node.yaml b/tests/test_utils/recipes/gb200/gpt-1node.yaml index 9b009735963..66f26ce719b 100644 --- a/tests/test_utils/recipes/gb200/gpt-1node.yaml +++ b/tests/test_utils/recipes/gb200/gpt-1node.yaml @@ -70,27 +70,27 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [mr-github, mr-github-slim] + scope: [L0, L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_reruns_persistent_1_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node] products: - environment: [dev] - scope: [mr-github, mr-github-slim] + scope: [L0, L1] platforms: [dgx_gb200] ####################################################################### # tp1_pp1 variants # @@ -98,17 +98,17 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp1_pp2 variants # @@ -116,12 +116,12 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] ####################################################################### # tp1_pp4 variants # @@ -129,57 +129,57 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp1 variants # @@ -187,22 +187,22 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp2 (no cp) variants # @@ -210,42 +210,42 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_mla_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp2_cp2 variants (PP reduced 2→1 for 1-node) # @@ -253,47 +253,47 @@ products: - test_case: [gpt3_mcore_te_tp2_pp2_cp2_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] ####################################################################### # tp4_pp1 variants # @@ -301,17 +301,17 @@ products: - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp4_pp2 variant (PP reduced 2→1 for 1-node) # @@ -319,5 +319,5 @@ products: - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml index fb02a6f167d..c650752b104 100644 --- a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml +++ b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml @@ -43,5 +43,5 @@ products: - test_case: [gpt_583m_perf_gb200_4gpu] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt-perf.yaml b/tests/test_utils/recipes/gb200/gpt-perf.yaml index 9bbd6cf33b7..e4a2b439b45 100644 --- a/tests/test_utils/recipes/gb200/gpt-perf.yaml +++ b/tests/test_utils/recipes/gb200/gpt-perf.yaml @@ -45,5 +45,5 @@ products: - test_case: [gpt_16b_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt.yaml b/tests/test_utils/recipes/gb200/gpt.yaml index a7ea92e8b99..4208af2425f 100644 --- a/tests/test_utils/recipes/gb200/gpt.yaml +++ b/tests/test_utils/recipes/gb200/gpt.yaml @@ -73,35 +73,45 @@ products: - test_case: [gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp2] products: - environment: [dev] - scope: [nightly, L0-smoke] + scope: [L3] + cadence: [nightly] + platforms: [dgx_gb200] + - environment: [dev] + scope: [L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp2_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp1_pp4] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp4_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp4_pp1_resume_torch] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] ####################################################################### # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # @@ -110,22 +120,28 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [nightly, L0-smoke] + scope: [L3] + cadence: [nightly] + platforms: [dgx_gb200] + - environment: [dev] + scope: [L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: @@ -134,47 +150,56 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] # products: @@ -184,152 +209,181 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_gdn] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp1_pp4_memory_speed] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp4_pp1_memory_speed] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] ####################################################################### # Super important mr, mr-github tests that run for DEV per mr, mr-github # @@ -337,35 +391,39 @@ products: - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_weekly_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] products: - environment: [dev] - scope: [weekly] + scope: [L4] + cadence: [weekly] platforms: [dgx_gb200] - test_case: [gpt3_weekly_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] products: - environment: [dev] - scope: [weekly] + scope: [L4] + cadence: [weekly] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml index f34b66dae5c..d8f01ae20cc 100644 --- a/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml +++ b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml @@ -43,5 +43,5 @@ products: - test_case: [hybrid_nanov3_3b_perf_gb200_4gpu] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf.yaml b/tests/test_utils/recipes/gb200/hybrid-perf.yaml index 78585557c42..2c7ac486558 100644 --- a/tests/test_utils/recipes/gb200/hybrid-perf.yaml +++ b/tests/test_utils/recipes/gb200/hybrid-perf.yaml @@ -44,5 +44,5 @@ products: - test_case: [hybrid_2b_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe-1node.yaml b/tests/test_utils/recipes/gb200/moe-1node.yaml index 54a674b9a6f..8b9523c1689 100644 --- a/tests/test_utils/recipes/gb200/moe-1node.yaml +++ b/tests/test_utils/recipes/gb200/moe-1node.yaml @@ -64,22 +64,22 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2_1node] products: - environment: [dev] - scope: [mr-github, mr-github-slim] + scope: [L0, L1] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node] products: - environment: [dev] - scope: [mr-github, mr-github-slim] + scope: [L0, L1] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] ####################################################################### # tp1_pp2 MoE variants # @@ -87,7 +87,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp2_pp1 MoE variants # @@ -95,27 +95,27 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node] products: - environment: [dev] - scope: [mr-github] + scope: [L1] platforms: [dgx_gb200] ####################################################################### # ep8 variant (EP reduced 8→4 for 1-node) # @@ -123,7 +123,7 @@ products: - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] ####################################################################### # tp4_ep2_etp2_pp2 variant (PP reduced 2→1 for 1-node) # @@ -131,5 +131,5 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph_1node] products: - environment: [dev] - scope: [mr-github-temp-disabled] + scope: [L1-temp-disabled] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml b/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml index d1d6ea865b4..7ebbeede2c9 100644 --- a/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/gb200/moe-dynamic-inference.yaml @@ -61,5 +61,5 @@ products: - test_case: [gpt_dynamic_inference_tp2_pp2_ep2_gptoss_20b_swa] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe.yaml b/tests/test_utils/recipes/gb200/moe.yaml index 8f934ae13ab..3e2004901bd 100644 --- a/tests/test_utils/recipes/gb200/moe.yaml +++ b/tests/test_utils/recipes/gb200/moe.yaml @@ -67,37 +67,47 @@ products: - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly, L0-smoke] + scope: [L3] + cadence: [nightly] + platforms: [dgx_gb200] + - environment: [dev] + scope: [L0-smoke] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] # products: # non-determinism: #478 @@ -113,7 +123,8 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] # products: @@ -133,22 +144,26 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -168,27 +183,30 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: - environment: [dev] - scope: [mr, mr-slim] + scope: [L0, L2] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] products: - environment: [dev] - scope: [mr, mr-slim] + scope: [L0, L2] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: @@ -214,7 +232,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: @@ -225,5 +243,5 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/h100/bert.yaml b/tests/test_utils/recipes/h100/bert.yaml index 0f638a83151..e307f5dba70 100644 --- a/tests/test_utils/recipes/h100/bert.yaml +++ b/tests/test_utils/recipes/h100/bert.yaml @@ -61,35 +61,45 @@ products: - test_case: [bert_mcore_tp2_pp2] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_local_spec] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp2_pp2_resume_torch_dist_local_spec] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp1_pp2] products: - environment: [dev] - scope: [nightly, L0-smoke] + scope: [L3] + cadence: [nightly] + platforms: [dgx_h100] + - environment: [dev] + scope: [L0-smoke] platforms: [dgx_h100] - test_case: [bert_mcore_tp1_pp4_vp2] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [bert_mcore_tp4_pp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/flextron.yaml b/tests/test_utils/recipes/h100/flextron.yaml index 4b89f78037e..9f04fb4df76 100644 --- a/tests/test_utils/recipes/h100/flextron.yaml +++ b/tests/test_utils/recipes/h100/flextron.yaml @@ -60,5 +60,6 @@ products: - test_case: [hybrid_flextron_nightly_tp2_pp1_ep2_dgx_h100_1N8G] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml index fdc96221e44..ed00906a603 100644 --- a/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml @@ -60,32 +60,32 @@ products: - test_case: [gpt_dynamic_inference_tp8_pp1_dp1_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp8_dp1_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_dp2_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_prefix_caching_longest_prefix_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_dp8_583m_prefix_caching_round_robin_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] # Throughput coverage for this 583M dynamic-batching DP=8 config now lives at # tests/performance_tests/test_cases/gpt/gpt_583m_perf/ — see diff --git a/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml index 43661c16cd3..244ee6025fb 100644 --- a/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml @@ -65,116 +65,116 @@ products: - test_case: [gpt_dynamic_inference_tp8_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_flashinfer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_chunked_prefill] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_cuda_graphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_chunked_prefill_cuda_graphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] # ===== Round 2 (Tier 2) - parallelism x feature, 3-way combos, sampling, memory variants ===== - test_case: [gpt_dynamic_inference_tp2_pp2_583m_prefix_caching] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp8_pp1_583m_prefix_caching] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp8_583m_prefix_caching] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_chunked_prefill] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_583m_flashinfer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_cuda_graphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill_cuda_graphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_583m_prefix_caching_cuda_graphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_chunked_prefill_flashinfer] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_top_p_sampling] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_stop_words] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_top_n_logprobs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_prefix_caching_lru] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp1_pp1_583m_uvm_level1] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-grads.yaml b/tests/test_utils/recipes/h100/gpt-grads.yaml index c8ff49c47f6..d2f20d20966 100644 --- a/tests/test_utils/recipes/h100/gpt-grads.yaml +++ b/tests/test_utils/recipes/h100/gpt-grads.yaml @@ -64,5 +64,5 @@ products: - test_case: [gpt3_mcore_reruns_resume_check_grads] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-grpo.yaml b/tests/test_utils/recipes/h100/gpt-grpo.yaml index b2c2ce5bb99..ddae6dfdca2 100644 --- a/tests/test_utils/recipes/h100/gpt-grpo.yaml +++ b/tests/test_utils/recipes/h100/gpt-grpo.yaml @@ -59,17 +59,17 @@ products: - test_case: [gpt_grpo_basic_function] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_throughput] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_grpo_tp4_pp1_dp2_8b_throughput_github] products: diff --git a/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml b/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml index 7002f28d8c6..3baa92ef0c8 100644 --- a/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml +++ b/tests/test_utils/recipes/h100/gpt-inference-server-smoke.yaml @@ -45,5 +45,5 @@ products: - test_case: [gpt_inference_server_smoke_tp1_pp1_dp8_583m] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-offline-inference.yaml b/tests/test_utils/recipes/h100/gpt-offline-inference.yaml index 8451993b19f..744e2bdb141 100644 --- a/tests/test_utils/recipes/h100/gpt-offline-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-offline-inference.yaml @@ -58,15 +58,15 @@ products: - test_case: [gpt_offline_inference_sync_tp1_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_offline_inference_sync_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_offline_inference_async_tp1_pp1_dp8_583m_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml b/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml index 6d9cc4e948f..249b1588982 100644 --- a/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml +++ b/tests/test_utils/recipes/h100/gpt-perf-dp8.yaml @@ -41,5 +41,5 @@ products: - test_case: [gpt_583m_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-perf.yaml b/tests/test_utils/recipes/h100/gpt-perf.yaml index 88817a9b2df..3b9df4ab125 100644 --- a/tests/test_utils/recipes/h100/gpt-perf.yaml +++ b/tests/test_utils/recipes/h100/gpt-perf.yaml @@ -41,5 +41,5 @@ products: - test_case: [gpt_16b_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt-static-inference.yaml b/tests/test_utils/recipes/h100/gpt-static-inference.yaml index c904a533bfa..1ef698708d2 100644 --- a/tests/test_utils/recipes/h100/gpt-static-inference.yaml +++ b/tests/test_utils/recipes/h100/gpt-static-inference.yaml @@ -60,20 +60,20 @@ products: - test_case: [gpt_static_inference_tp1_pp1_583m_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_583m_cudagraphs] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_583m_fp8_cudagraphs] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp1_pp1_16b_multiprompt_tokensmatch] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/gpt.yaml b/tests/test_utils/recipes/h100/gpt.yaml index 3e88e325048..e8365638b3c 100644 --- a/tests/test_utils/recipes/h100/gpt.yaml +++ b/tests/test_utils/recipes/h100/gpt.yaml @@ -65,45 +65,57 @@ products: - test_case: [gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp1_pp2] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp1_pp2_resume_torch_dist] products: - environment: [dev, lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp1_pp4] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp1_pp4_resume_torch_dist] products: - environment: [dev, lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp4_pp1_resume_torch] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [gpt3_mcore_tp4_pp1_resume_torch_dist] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] ####################################################################### # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # @@ -112,29 +124,32 @@ products: - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [mr, mr-github, L0-smoke] + scope: [L1, L2, L0-smoke] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github-broken] + scope: [L2, mr-github-broken] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] # products: # - environment: [dev] @@ -144,66 +159,72 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # outdated TE: #501 - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #436 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # non-determinism: #437 - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] # products: # - environment: [dev] @@ -214,247 +235,271 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: # - environment: [dev] # scope: [mr] # platforms: [dgx_h100] # Hangs: #513 - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_gdn] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_sync] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_async] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_gdn_no_nvrx_async_mcore] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_dsa] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader] products: # - environment: [dev] # scope: [mr] # Hangs: #513 # platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_7b_tp1_pp4_memory_speed] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # OOM: #434 - test_case: [gpt3_7b_tp4_pp1_memory_speed] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # OOM: #434 - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] # Outdated: #502 @@ -470,7 +515,7 @@ products: - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [mr, mr-github-broken] + scope: [L2, mr-github-broken] platforms: [dgx_h100] # - environment: [lts] # scope: [nightly] @@ -484,40 +529,46 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_weekly_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] products: - environment: [dev] - scope: [weekly] + scope: [L4] + cadence: [weekly] platforms: [dgx_h100] - test_case: [gpt3_weekly_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] products: - environment: [dev] - scope: [weekly] + scope: [L4] + cadence: [weekly] platforms: [dgx_h100] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_uninstall_te] # products: diff --git a/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml b/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml index 5c81e98b49d..6223fc2e32b 100644 --- a/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml +++ b/tests/test_utils/recipes/h100/hybrid-perf-ep8.yaml @@ -40,5 +40,5 @@ products: - test_case: [hybrid_nanov3_3b_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/hybrid-perf.yaml b/tests/test_utils/recipes/h100/hybrid-perf.yaml index 11835061ec2..56b4f0abcf8 100644 --- a/tests/test_utils/recipes/h100/hybrid-perf.yaml +++ b/tests/test_utils/recipes/h100/hybrid-perf.yaml @@ -40,5 +40,5 @@ products: - test_case: [hybrid_2b_perf] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml b/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml index 56f56c17a82..f72a1595f8a 100644 --- a/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml @@ -60,20 +60,20 @@ products: - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_chunked_prefill] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_flashinfer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [hybrid_dynamic_inference_tp1_pp1_dp8_583m_mamba_bf16_states] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/mamba-static-inference.yaml b/tests/test_utils/recipes/h100/mamba-static-inference.yaml index bafabf084fc..b9e42668d9f 100644 --- a/tests/test_utils/recipes/h100/mamba-static-inference.yaml +++ b/tests/test_utils/recipes/h100/mamba-static-inference.yaml @@ -65,5 +65,5 @@ products: - test_case: [hybrid_static_inference_tp1_pp1_2B_cudagraphs] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dg x_h100] diff --git a/tests/test_utils/recipes/h100/mamba.yaml b/tests/test_utils/recipes/h100/mamba.yaml index 77331a751e4..0ec42cc8a22 100644 --- a/tests/test_utils/recipes/h100/mamba.yaml +++ b/tests/test_utils/recipes/h100/mamba.yaml @@ -60,7 +60,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -68,7 +68,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp4_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -76,7 +76,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp1_pp2_vpp2_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -84,7 +84,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp2_pp1_cp1_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] @@ -92,7 +92,7 @@ products: - test_case: [hybrid_mr_mcore_te_tp2_pp1_cp4_dgx_a100_1N8G] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - environment: [lts] # disabled until triton is bumped # scope: [nightly] diff --git a/tests/test_utils/recipes/h100/module_performance.yaml b/tests/test_utils/recipes/h100/module_performance.yaml index 96e109498da..577907f2362 100644 --- a/tests/test_utils/recipes/h100/module_performance.yaml +++ b/tests/test_utils/recipes/h100/module_performance.yaml @@ -50,5 +50,5 @@ products: - test_case: [moe_perf] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index 1ecc7247c72..9934a470d14 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -60,12 +60,12 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_zmq] products: - environment: [dev] - scope: [mr] #[mr, mr-github] - broken on github after NGC PyTorch26.02 + scope: [L2] #[mr, mr-github] - broken on github after NGC PyTorch26.02 platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [mr] #[mr, mr-github] - broken on github after NGC PyTorch26.02 + scope: [L2] #[mr, mr-github] - broken on github after NGC PyTorch26.02 platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_zmq_suspend_resume] products: diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml index 889542638e4..57feb6cd6da 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml @@ -65,20 +65,20 @@ products: - test_case: [gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp2_pp2_ep2_gptoss_20b_swa] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_ep4_16B_prefix_caching] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [gpt_dynamic_inference_tp4_pp1_ep4_16B_chunked_prefill] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-grpo.yaml b/tests/test_utils/recipes/h100/moe-grpo.yaml index fcad7111f05..040d7e15047 100644 --- a/tests/test_utils/recipes/h100/moe-grpo.yaml +++ b/tests/test_utils/recipes/h100/moe-grpo.yaml @@ -59,5 +59,5 @@ products: - test_case: [gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-static-inference.yaml b/tests/test_utils/recipes/h100/moe-static-inference.yaml index e82b452b2b9..e3c5ac2bf48 100644 --- a/tests/test_utils/recipes/h100/moe-static-inference.yaml +++ b/tests/test_utils/recipes/h100/moe-static-inference.yaml @@ -60,15 +60,15 @@ products: - test_case: [gpt_static_inference_tp1_pp1_ep1_16B_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github-broken] + scope: [L2, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt_static_inference_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github-broken] + scope: [L2, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt_static_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github-broken] + scope: [L2, mr-github-broken] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml index 69b02ce639f..145005c87ce 100644 --- a/tests/test_utils/recipes/h100/moe.yaml +++ b/tests/test_utils/recipes/h100/moe.yaml @@ -65,48 +65,60 @@ products: - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] # products: # non-determinism: #478 # - environment: [dev, lts] @@ -121,7 +133,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] # products: @@ -141,22 +153,22 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp1_pp1_te_4experts_groupedGEMM_op_fuser] products: - environment: [dev] - scope: [mr, mr-github, L0-smoke] + scope: [L1, L2, L0-smoke] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4] products: @@ -164,11 +176,12 @@ products: # scope: [mr] # platforms: [dgx_h100] # hang: #513 - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -178,29 +191,30 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + scope: [L3] + cadence: [nightly] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: - environment: [dev] - scope: [mr, mr-github, mr-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] products: - environment: [dev] - scope: [mr, mr-github, mr-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: @@ -215,7 +229,7 @@ products: - test_case: [deepseek_proxy_fsdp_ep2_fsdp2_ep_overlap] products: - environment: [dev] - scope: [mr, mr-github] + scope: [L1, L2] platforms: [dgx_h100] ####################################################################### # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # @@ -236,7 +250,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [L0, L1, L2] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: diff --git a/tests/test_utils/recipes/h100/multimodal-llava.yaml b/tests/test_utils/recipes/h100/multimodal-llava.yaml index 0c6ff97a002..42337ac1206 100644 --- a/tests/test_utils/recipes/h100/multimodal-llava.yaml +++ b/tests/test_utils/recipes/h100/multimodal-llava.yaml @@ -63,10 +63,10 @@ products: - test_case: [multimodal_llava_mcore_te_tp1_pp1] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] - test_case: [multimodal_llava_mcore_te_tp4_sp_cp2] products: - environment: [dev] - scope: [mr] + scope: [L2] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/t5.yaml b/tests/test_utils/recipes/h100/t5.yaml index 039a7beafb6..917dec3dc3d 100644 --- a/tests/test_utils/recipes/h100/t5.yaml +++ b/tests/test_utils/recipes/h100/t5.yaml @@ -61,55 +61,69 @@ products: - test_case: [t5_11b_mcore_tp4_pp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp4_pp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_tp4_pp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_tp4_pp1_resume_torch_dist] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_h100] - test_case: [t5_mcore_te_tp1_pp1_vp1_resume_torch] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_te_tp2_pp1_vp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_te_tp2_pp1_vp1_sequence_parallel] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp1_pp1_vp1] products: - environment: [dev] - scope: [nightly, L0-smoke] + scope: [L3] + cadence: [nightly] + platforms: [dgx_a100, dgx_h100] + - environment: [dev] + scope: [L0-smoke] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp1_pp1_vp1_resume_torch] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] - test_case: [t5_mcore_tp2_pp1_vp1] products: - environment: [dev] - scope: [nightly] + scope: [L3] + cadence: [nightly] platforms: [dgx_a100, dgx_h100] diff --git a/tools/trigger_internal_ci.md b/tools/trigger_internal_ci.md index 8d3a8577537..fb6ce63906c 100644 --- a/tools/trigger_internal_ci.md +++ b/tools/trigger_internal_ci.md @@ -37,7 +37,7 @@ python -m pip install python-gitlab python tools/trigger_internal_ci.py \ --gitlab-origin gitlab \ [--access-token glpat-] \ - [--functional-test-scope mr] \ + [--functional-test-scope L1,L2] \ [--functional-test-repeat 5] \ [--functional-test-cases all] \ [--functional-test-name release-testing/mcore-vX.Y.Z] \ @@ -49,7 +49,7 @@ python tools/trigger_internal_ci.py \ |---|---|---| | `--gitlab-origin` | *(required)* | Git remote name for the internal GitLab | | `--access-token` | `$GITLAB_TOKEN` | Personal access token with `api` scope | -| `--functional-test-scope` | `mr` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | +| `--functional-test-scope` | `L1,L2` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | | `--functional-test-repeat` | `5` | `FUNCTIONAL_TEST_REPEAT` pipeline variable | | `--functional-test-cases` | `all` | `FUNCTIONAL_TEST_CASES` pipeline variable | | `--functional-test-name` | commit SHA | `FUNCTIONAL_TEST_NAME` pipeline variable — names the run for `pre-release`/`release` scopes (used as the run name and W&B experiment). | diff --git a/tools/trigger_internal_ci.py b/tools/trigger_internal_ci.py index d46a2f6436c..33b87cb3713 100644 --- a/tools/trigger_internal_ci.py +++ b/tools/trigger_internal_ci.py @@ -40,6 +40,10 @@ "INTEGRATION_TEST": "no", } +# No trigger-side scope aliases are applied. The internal GitLab CI / recipe +# parser resolves legacy scope values server-side; bare `mr` maps to `L2`. +SCOPE_ALIASES = {} + # Scopes whose recipes run full convergence/checkpointing workloads and need a # long wall-clock budget. The default short-scope time limit is left untouched. LONG_RUNNING_SCOPES = ("release", "weekly") @@ -48,6 +52,11 @@ logger = logging.getLogger(__name__) +def normalize_scope(scope): + """Map a legacy FUNCTIONAL_TEST_SCOPE value to its L-tier equivalent.""" + return SCOPE_ALIASES.get(scope, scope) + + def resolve_time_limit(scope, override): """Resolve the FUNCTIONAL_TEST_TIME_LIMIT value for a functional test scope. @@ -150,8 +159,9 @@ def main(): ) parser.add_argument( "--functional-test-scope", - default="mr", - help="FUNCTIONAL_TEST_SCOPE pipeline variable (default: mr)", + default="L1,L2", + help="FUNCTIONAL_TEST_SCOPE pipeline variable (default: L1,L2). " + "Legacy 'mr' resolves server-side to L2.", ) parser.add_argument( "--functional-test-repeat", @@ -220,9 +230,17 @@ def main(): git_push(args.gitlab_origin, target_branch, dry_run=args.dry_run) + functional_test_scope = normalize_scope(args.functional_test_scope) + if functional_test_scope != args.functional_test_scope: + logger.info( + "Normalized FUNCTIONAL_TEST_SCOPE '%s' -> '%s'", + args.functional_test_scope, + functional_test_scope, + ) + pipeline_vars = { **PIPELINE_VARIABLES_FIXED, - "FUNCTIONAL_TEST_SCOPE": args.functional_test_scope, + "FUNCTIONAL_TEST_SCOPE": functional_test_scope, "FUNCTIONAL_TEST_REPEAT": str(args.functional_test_repeat), "FUNCTIONAL_TEST_CASES": args.functional_test_cases, }