diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 42c22191acb2..4b9176307841 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -107,8 +107,11 @@ jobs: # caller that declares its runners as a matrix cannot, since job-level `if` # sees github/needs/vars/inputs but not `matrix`. Callers that never declare # the input read back null and match every runner_config. + # Temporarily skip the unavailable GB300 runner before GitHub tries to + # allocate it. Covers both callers: pr-test and nightly. if: | always() && + inputs.runner_config != '4-gpu-gb300' && (fromJson(inputs.caller_inputs).runner_filter == null || fromJson(inputs.caller_inputs).runner_filter == '' || fromJson(inputs.caller_inputs).runner_filter == 'all' || diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index a3e4e2c5294e..325c7293a9c4 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -23,7 +23,7 @@ on: - '2-gpu-large' - '4-gpu-h100' - '4-gpu-b200' - - '4-gpu-gb300' + # - '4-gpu-gb300' # Temporarily disabled while the GB300 runner is unavailable. - '8-gpu-h200' - '8-gpu-b200' - 'diffusion' diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml index 7037bce67b6f..284eda2969f7 100644 --- a/.github/workflows/release-whl-deepgemm.yml +++ b/.github/workflows/release-whl-deepgemm.yml @@ -114,10 +114,11 @@ jobs: # - arch_label: sm120 # runner: 1-gpu-5090 # wheel_arch: x86_64 - - arch_label: sm100-aarch64 - runner: 4-gpu-gb300 - wheel_arch: aarch64 - timeout: 180 + # Temporarily disabled while the GB300 runner is unavailable; the aarch64 wheel still publishes untested. + # - arch_label: sm100-aarch64 + # runner: 4-gpu-gb300 + # wheel_arch: aarch64 + # timeout: 180 runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.timeout }} steps: diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index 74190784431f..e3c83c8bc8f3 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -119,7 +119,8 @@ jobs: rerun-test-cuda: name: rerun-test-cuda (partition ${{ matrix.partition }}) needs: prepare-partitions - if: inputs.mode == 'cuda' + # Temporarily reject reruns targeting the unavailable GB300 runner. + if: inputs.mode == 'cuda' && inputs.runs_on != '4-gpu-gb300' strategy: fail-fast: false matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }} diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index abfe8d54fa25..570661c2e470 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -24,6 +24,10 @@ # (single path component under that dir, no whitespace/operators). _ALLOWED_INSTALL_SCRIPT = re.compile(r"^scripts/ci/cuda/[\w.-]+\.sh$") +# Temporarily disabled while their runners are unavailable. rerun-test.yml +# skips these too; rejecting here gives a clear reply instead of a skipped run. +_DISABLED_RUNNER_CONFIGS = {"4-gpu-gb300"} + # Configuration PERMISSIONS_FILE_PATH = ".github/CI_PERMISSIONS.json" TEST_GROUPS_FILE_PATH = "scripts/ci/rerun_test_groups.json" @@ -991,6 +995,12 @@ def _resolve_runner_config(rc, full_path, suite): f"— not in scripts/ci/runner_configs.yml.\n\n" f"Known runner_configs: {known}", ) + if rc in _DISABLED_RUNNER_CONFIGS: + return _dispatch_err( + suite, + f"runner_config `{rc}` in `{full_path}` is temporarily disabled " + f"while its runners are unavailable.", + ) install_script = cfg["install"] if not _ALLOWED_INSTALL_SCRIPT.match(install_script): return _dispatch_err(