[CI] 1-gpu-small: Move 1-gpu-large tests that pass on the 5090

.agents/skills/add-jit-kernel/SKILL.md

@@ -554,23 +554,24 @@ if torch.cuda.get_device_capability()[0] < 9:
 
 JIT kernel correctness tests and benchmarks live under `test/registered/kernels/ops/<group>/` and `test/registered/kernels/benchmark/<group>/`, mirroring the wrapper's group under `python/sglang/kernels/ops/` (NOT inside the `sglang` package -- a `register_*_ci(...)` call anywhere under `python/sglang/` is rejected by the `check-no-registered-tests-in-package` pre-commit hook). Only their test-only helpers (e.g. `benchmark/marker.py`) stay alongside the kernel source under `python/sglang/kernels/jit/` and are imported by absolute path. **CI does not run `pytest` in those directories directly.** The unified runner `test/run_suite.py` discovers every `test_*.py` and `bench_*.py` under `test/registered/`, collects `register_*_ci(...)` calls by **statically parsing each file's AST**, and executes the selected suite. Every test file must register at least one CUDA entry or the collector fails its sanity check.
 
-- **PR / per-commit CUDA suites** (see `test/run_suite.py` → `PER_COMMIT_SUITES`): JIT unit tests use `base-b-kernel-unit-test-1-gpu-large` on H100 and `base-b-kernel-unit-test-4-gpu-b200` on B200/SM100 paths (see `.github/workflows/pr-test-jit-kernel.yml`). Multi-GPU JIT tests use `base-b-kernel-unit-test-8-gpu-h200`.
+- **PR / per-commit CUDA suites** (see `test/run_suite.py` → `PER_COMMIT_SUITES`): JIT unit tests use `base-b-kernel-unit-test-1-gpu-small` on the RTX 5090 (SM120) by default, `base-b-kernel-unit-test-1-gpu-large` on H100 only for SM90-only kernels (or shapes that exceed SM120's ~99 KB shared memory), and `base-b-kernel-unit-test-4-gpu-b200` on B200/SM100 paths (see `.github/workflows/pr-test-jit-kernel.yml`). Multi-GPU JIT tests use `base-b-kernel-unit-test-8-gpu-h200`.
 - **Nightly kernel suite**: register with `stage="nightly"` plus the `runner_config` of the machine it needs (e.g. `1-gpu-large`), giving the `nightly-test-1-gpu-large` suite. `.github/workflows/nightly-test-nvidia.yml` sets `SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1` for the whole nightly run, so the expanded parameter grids apply automatically (see `python/sglang/kernels/jit/utils/common.py` → `should_run_full_tests` / `get_ci_test_range`). There is no separate kernel-only nightly job: every nightly test on one machine type shares that machine's suite.
 
 Registration pattern (module level, **literal** `est_time`, `stage`, and `runner_config` values — required for AST parsing):
 
 ```python
 from sglang.test.ci.ci_register import register_cuda_ci
 
-register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
+register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-small")
+# SM90-only kernels register on runner_config="1-gpu-large" instead.
 # Optional B200/SM100 registration for tests that cover Blackwell-specific code paths
 # register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
 # Optional second registration: same file also runs nightly, same form,
 # stage is just "nightly" there (and no `nightly=True`)
 # register_cuda_ci(est_time=120, stage="nightly", runner_config="1-gpu-large")
 ```
 
-CI generates the suite name as `{stage}-test-{runner_config}`, so `stage="base-b-kernel-unit", runner_config="1-gpu-large"` becomes the `base-b-kernel-unit-test-1-gpu-large` suite you pass to `run_suite.py` below — don't put the `-test-` infix in `register_cuda_ci`. Nightly uses the same shape with `stage="nightly"`; the single-string `suite=` form is left only for `stress` and non-CUDA pools.
+CI generates the suite name as `{stage}-test-{runner_config}`, so `stage="base-b-kernel-unit", runner_config="1-gpu-small"` becomes the `base-b-kernel-unit-test-1-gpu-small` suite you pass to `run_suite.py` below — don't put the `-test-` infix in `register_cuda_ci`. Nightly uses the same shape with `stage="nightly"`; the single-string `suite=` form is left only for `stress` and non-CUDA pools.
 
 Keep `est_time`, `stage`, `runner_config`, and `suite` as literal values. `run_suite.py` collects them from the file AST, so computed values and helper wrappers can break CI discovery.
 
@@ -579,7 +580,7 @@ Use `register_cuda_ci(..., disabled="reason")` if the file must stay in-tree but
 **Run like CI** (from repo root):
 
 ```bash
-(cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-1-gpu-large)
+(cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-1-gpu-small)
 # For B200/SM100-specific coverage:
 (cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200)
 ```
@@ -594,7 +595,7 @@ import torch
 from sglang.kernels.ops.elementwise.scale import scale
 from sglang.test.ci.ci_register import register_cuda_ci
 
-register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
+register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-small")
 
 
 @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@@ -639,7 +640,7 @@ if __name__ == "__main__":
 
 ## Step 5: Add a benchmark (required)
 
-Benchmarks are `bench_*.py` files under `test/registered/kernels/benchmark/<group>/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-test-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large`).
+Benchmarks are `bench_*.py` files under `test/registered/kernels/benchmark/<group>/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-test-1-gpu-small`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-small`); use `base-b-kernel-benchmark-test-1-gpu-large` only for SM90-only kernels.
 
 Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md).
 
@@ -668,7 +669,7 @@ from sglang.kernels.jit.benchmark.utils import create_random
 from sglang.kernels.ops.elementwise.scale import scale as jit_scale
 from sglang.test.ci.ci_register import register_cuda_ci
 
-register_cuda_ci(est_time=6, stage="base-b-kernel-benchmark", runner_config="1-gpu-large")
+register_cuda_ci(est_time=6, stage="base-b-kernel-benchmark", runner_config="1-gpu-small")
 
 
 @torch.compile()
@@ -722,7 +723,7 @@ python test/registered/kernels/benchmark/elementwise/bench_scale.py
 Run the benchmark suite the way CI does:
 
 ```bash
-cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large
+cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-small
 ```
 
 ---

.agents/skills/ci-workflow-guide/SKILL.md

@@ -283,9 +283,11 @@ Large suites are split across matrix jobs using the **LPT (Longest Processing Ti
 | `base-b-test-1-gpu-large` | 14 | `1-gpu-h100` | dynamic (3 or 14) |
 | `base-b-test-2-gpu-large` | 4 | `2-gpu-h100` | — |
 | `base-b-test-4-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
+| `base-b-kernel-unit-test-1-gpu-small` | 4 | `1-gpu-5090` | — |
 | `base-b-kernel-unit-test-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
 | `base-b-kernel-unit-test-4-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
 | `base-b-kernel-unit-test-8-gpu-h200` | 1 (no matrix) | `8-gpu-h200` | — |
+| `base-b-kernel-benchmark-test-1-gpu-small` | 1 (no matrix) | `1-gpu-5090` | — |
 | `base-b-kernel-benchmark-test-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
 | `base-c-test-4-gpu-h100` | 3 | `4-gpu-h100` | — |
 | `base-c-test-8-gpu-h200` | 4 | `8-gpu-h200` | — |

.agents/skills/write-sglang-test/SKILL.md

@@ -105,9 +105,9 @@ Use the lightest suite that meets your test's needs:
 - **No GPU required** → `base-a-test-cpu`
 - **Most small GPU tests** → `base-b-test-1-gpu-small` (default choice)
 - **Need H100 memory or Hopper features** → `base-b-test-1-gpu-large`
-- **JIT kernel correctness** → `base-b-kernel-unit-test-1-gpu-large`
+- **JIT kernel correctness** → `base-b-kernel-unit-test-1-gpu-small` (default; SM90-only kernels → `base-b-kernel-unit-test-1-gpu-large`)
 - **JIT kernel correctness for B200 / SM100 paths** → `base-b-kernel-unit-test-4-gpu-b200`
-- **JIT kernel benchmarks** → `base-b-kernel-benchmark-test-1-gpu-large`
+- **JIT kernel benchmarks** → `base-b-kernel-benchmark-test-1-gpu-small` (default; SM90-only kernels → `base-b-kernel-benchmark-test-1-gpu-large`)
 - **Multi-GPU** → only when the test actually needs multiple GPUs
 
 ---
@@ -339,12 +339,12 @@ They are ordinary registered tests; only their stage differs:
 from sglang.test.ci.ci_register import register_cuda_ci
 
 # Correctness tests in test/registered/kernels/ops/<group>/
-register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
+register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-small")
 register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
 register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="8-gpu-h200")
 
 # Benchmarks in test/registered/kernels/benchmark/<group>/
-register_cuda_ci(est_time=6, stage="base-b-kernel-benchmark", runner_config="1-gpu-large")
+register_cuda_ci(est_time=6, stage="base-b-kernel-benchmark", runner_config="1-gpu-small")
 
 # Optional nightly registration — same form, stage is just "nightly"
 register_cuda_ci(est_time=120, stage="nightly", runner_config="1-gpu-large")

.github/workflows/pr-test-jit-kernel.yml

@@ -59,14 +59,23 @@ jobs:
         include:
           - suite: unit-test-1-gpu-large
             runner_config: 1-gpu-large
+            suite_args: --fork-worker-batch-size 20
+          - suite: unit-test-1-gpu-small
+            runner_config: 1-gpu-small
             shard: " (0)"
-            suite_args: --auto-partition-id 0 --auto-partition-size 2 --fork-worker-batch-size 20
-          - suite: unit-test-1-gpu-large
-            runner_config: 1-gpu-large
+            suite_args: --auto-partition-id 0 --auto-partition-size 4 --fork-worker-batch-size 20
+          - suite: unit-test-1-gpu-small
+            runner_config: 1-gpu-small
             shard: " (1)"
-            suite_args: --auto-partition-id 1 --auto-partition-size 2 --fork-worker-batch-size 20
+            suite_args: --auto-partition-id 1 --auto-partition-size 4 --fork-worker-batch-size 20
+          - suite: unit-test-1-gpu-small
+            runner_config: 1-gpu-small
+            shard: " (2)"
+            suite_args: --auto-partition-id 2 --auto-partition-size 4 --fork-worker-batch-size 20
           - suite: unit-test-1-gpu-small
             runner_config: 1-gpu-small
+            shard: " (3)"
+            suite_args: --auto-partition-id 3 --auto-partition-size 4 --fork-worker-batch-size 20
           - suite: unit-test-4-gpu-b200
             runner_config: 4-gpu-b200
           - suite: unit-test-8-gpu-h200
@@ -75,6 +84,8 @@ jobs:
             skip_health_check: true
           - suite: benchmark-test-1-gpu-large
             runner_config: 1-gpu-large
+          - suite: benchmark-test-1-gpu-small
+            runner_config: 1-gpu-small
     runs-on: ${{ fromJson(inputs.runs_on_map)[matrix.runner_config] }}
     timeout-minutes: 240
     steps:

.github/workflows/pr-test-multimodal-gen.yml

@@ -215,7 +215,7 @@ jobs:
     if: |
       ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
       inputs.multimodal_gen == 'true'
-    runs-on: 1-gpu-h100
+    runs-on: 1-gpu-5090
     timeout-minutes: 90
     steps:
       - name: Checkout code
@@ -249,7 +249,7 @@ jobs:
         run: |
           python3 test/run_suite.py \
             --hw cuda \
-            --suite base-b-test-diffusion-bcg-1-gpu-h100 \
+            --suite base-b-test-diffusion-bcg-1-gpu-5090 \
             --timeout-per-file 3600
 
       - name: Upload BCG diffusion artifacts
@@ -465,7 +465,7 @@ jobs:
     if: |
       ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
       inputs.multimodal_gen == 'true'
-    runs-on: 1-gpu-h100
+    runs-on: 1-gpu-5090
     timeout-minutes: 120
     steps:
       - name: Checkout code
@@ -495,9 +495,47 @@ jobs:
         run: |
           python3 test/run_suite.py \
             --hw cuda \
-            --suite base-b-test-diffusion-unit-1-gpu-h100 \
+            --suite base-b-test-diffusion-unit-1-gpu-5090 \
             --timeout-per-file 3600
 
+  multimodal-gen-unit-test-h100:
+    # Only the unit files with SM90 / H100-only cases; the full suite runs on the 5090.
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 1-gpu-h100
+    timeout-minutes: 60
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: python/sglang/kernels/aot/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run H100-only diffusion unit tests
+        timeout-minutes: 20
+        run: |
+          python3 test/run_suite.py \
+            --hw cuda \
+            --suite base-b-test-diffusion-unit-1-gpu-h100 \
+            --timeout-per-file 1200
+
   diffusion-coverage-check:
     needs: [multimodal-gen-test-1-gpu, multimodal-gen-test-2-gpu]
     if: |

python/sglang/multimodal_gen/test/server/gpu_cases.py

@@ -1559,6 +1559,13 @@ def _discover_unit_tests() -> list[str]:
 
 FILE_SUITES = {
     "unit": _discover_unit_tests(),
+    # The full unit suite runs on the 5090; these files also hold cases that only
+    # run on H100 (SM90 kernels, pre-Blackwell fallbacks, FA on SM90, >=48 GiB free).
+    "unit-h100": [
+        "../unit/test_kandinsky6_bf16_dtype.py",
+        "../unit/test_mxfp8_online_gemm.py",
+        "../unit/test_subblock_sparse_attention.py",
+    ],
     "component-accuracy": [
         "../single_test_file/component_accuracy/test_component_accuracy_1_gpu.py",
         "../single_test_file/component_accuracy/test_component_accuracy_2_gpu.py",
@@ -1678,7 +1685,7 @@ def _discover_unit_tests() -> list[str]:
     },
 }
 
-STRICT_SUITES = {"unit", "bcg-diffusion"}
+STRICT_SUITES = {"unit", "unit-h100", "bcg-diffusion"}
 COMPONENT_ACCURACY_SUITES = {
     "component-accuracy",
     "component-accuracy-1-gpu",

test/README.md

@@ -90,8 +90,8 @@ root and mirror the operator group under `python/sglang/kernels/ops/`. Helpers
 stay alongside the kernel source under `python/sglang/kernels/jit/` and are
 imported by absolute path:
 
-- Correctness tests: `test/registered/kernels/ops/<group>/test_*.py` → `base-b-kernel-unit-test-1-gpu-large`
-- Benchmarks: `test/registered/kernels/benchmark/<group>/bench_*.py` → `base-b-kernel-benchmark-test-1-gpu-large`
+- Correctness tests: `test/registered/kernels/ops/<group>/test_*.py` → `base-b-kernel-unit-test-1-gpu-small` (use `base-b-kernel-unit-test-1-gpu-large` only for SM90-only kernels or shapes that exceed the 5090's ~99 KB shared memory)
+- Benchmarks: `test/registered/kernels/benchmark/<group>/bench_*.py` → `base-b-kernel-benchmark-test-1-gpu-small` (use `base-b-kernel-benchmark-test-1-gpu-large` only for SM90-only kernels)
 
 ## Choosing a Suite
 
@@ -102,8 +102,8 @@ Use the lightest suite that meets your test's needs. Full suite tables are in th
 | No GPU required | `base-a-test-cpu` |
 | Small GPU (fits 5090, 32GB) | `base-b-test-1-gpu-small` (most tests go here) |
 | Large GPU memory or Hopper features | `base-b-test-1-gpu-large` |
-| JIT kernel correctness | `base-b-kernel-unit-test-1-gpu-large` |
-| JIT kernel benchmarks | `base-b-kernel-benchmark-test-1-gpu-large` |
+| JIT kernel correctness | `base-b-kernel-unit-test-1-gpu-small` (SM90-only kernels: `base-b-kernel-unit-test-1-gpu-large`) |
+| JIT kernel benchmarks | `base-b-kernel-benchmark-test-1-gpu-small` (SM90-only kernels: `base-b-kernel-benchmark-test-1-gpu-large`) |
 | Multi-GPU (2/4/8) | `base-b-test-2-gpu-large`, `base-c-test-*` |
 | Long-running or experimental | `nightly-*` suites |
 

test/registered/attention/test_chunk_gated_delta_rule.py

@@ -13,7 +13,7 @@
     register_xpu_ci,
 )
 
-register_cuda_ci(est_time=6, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=79, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=11, stage="stage-b", runner_config="1-gpu-large-amd")
 register_xpu_ci(est_time=900, suite="stage-b-test-1-gpu-xpu")
 

test/registered/attention/test_deterministic.py

@@ -24,7 +24,8 @@
     is_in_amd_ci,
 )
 
-register_cuda_ci(est_time=280, stage="base-b", runner_config="1-gpu-large")
+# The FA3 class (SM80/SM90 only) is in test_deterministic_fa3.py.
+register_cuda_ci(est_time=300, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=278, suite="stage-b-test-1-gpu-small-amd")
 register_xpu_ci(est_time=207, suite="stage-b-test-1-gpu-xpu")
 
@@ -47,22 +48,6 @@ def get_server_args(cls):
         return args
 
 
-@unittest.skipIf(_is_xpu, "CUDA runner only")
-@unittest.skipIf(is_in_amd_ci(), "Skip for AMD CI.")
-class TestFa3Deterministic(TestDeterministicBase):
-    # Test with fa3 attention backend
-    @classmethod
-    def get_server_args(cls):
-        args = COMMON_SERVER_ARGS
-        args.extend(
-            [
-                "--attention-backend",
-                "fa3",
-            ]
-        )
-        return args
-
-
 @unittest.skipIf(_is_xpu, "CUDA/AMD runner only")
 class TestTritonDeterministic(TestDeterministicBase):
     # Test with triton attention backend

test/registered/attention/test_deterministic_fa3.py

@@ -0,0 +1,21 @@
+"""FA3 class of test_deterministic.py, kept on H100 because FA3 is SM80/SM90 only."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_deterministic_utils import (
+    COMMON_SERVER_ARGS,
+    TestDeterministicBase,
+)
+
+register_cuda_ci(est_time=95, stage="base-b", runner_config="1-gpu-large")
+
+
+class TestFa3Deterministic(TestDeterministicBase):
+    @classmethod
+    def get_server_args(cls):
+        return COMMON_SERVER_ARGS + ["--attention-backend", "fa3"]
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/attention/test_gdn_fused_split_head_ratios.py

@@ -7,7 +7,7 @@
 )
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=6, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=6, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=6, suite="stage-b-test-1-gpu-small-amd")
 
 

test/registered/attention/test_gdn_noncontiguous_stride.py

@@ -14,7 +14,7 @@
 )
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=7, stage="stage-b", runner_config="1-gpu-large-amd")
 
 

test/registered/attention/test_gdn_prefill_layout.py

@@ -17,7 +17,7 @@
 from sglang.srt.model_executor.forward_batch_info import ForwardMode
 from sglang.test.ci.ci_register import register_cuda_ci
 
-register_cuda_ci(est_time=6, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=16, stage="base-b", runner_config="1-gpu-small")
 
 
 @unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA")

test/registered/attention/test_gemma4_unified_swa_virtual_ids.py

@@ -48,7 +48,7 @@
     popen_launch_server,
 )
 
-register_cuda_ci(est_time=72, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=72, stage="base-b", runner_config="1-gpu-small")
 
 # Healthy 0.865-0.880 (static pool 0.870); a died-mid-run server scores
 # 0.05-0.21. 1 sigma over 200 examples is ~0.024.

test/registered/attention/test_kda_kernels.py

@@ -21,7 +21,7 @@
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=25, stage="base-b", runner_config="1-gpu-large")
+register_cuda_ci(est_time=145, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=12, stage="stage-b", runner_config="1-gpu-large-amd")