test: skip GPU tests when CUDA is unavailable

.github/actions/detect-changes/action.yml

@@ -84,6 +84,7 @@ runs:
             - '.github/workflows/**'
             - '.github/actions/**'
             - '.mise.toml'
+            - '.mise.*.toml'
             - '.mise/tasks/**'
             - '.dockerignore'
             - 'containers/Dockerfile.test_ci'

.github/actions/setup-gpu-test-env/action.yml

@@ -118,8 +118,18 @@ runs:
         uv --no-config pip check --python "${python_bin}"
         "${python_bin}" -c \
           "from importlib.metadata import version; from nemo_safe_synthesizer.package_info import __version__; assert __version__ == version('nemo-safe-synthesizer')"
-        "${python_bin}" -c \
-          "import torch; print('cuda available:', torch.cuda.is_available()); print('device count:', torch.cuda.device_count()); assert torch.cuda.is_available()"
+        "${python_bin}" - <<'PY'
+        try:
+            import torch
+        except ImportError:
+            raise SystemExit("GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported.") from None
+
+        if not torch.cuda.is_available():
+            raise SystemExit("GPU CI requires CUDA, but torch.cuda.is_available() returned false.")
+
+        print("cuda available: true")
+        print("device count:", torch.cuda.device_count())
+        PY
 
         echo "UV_PROJECT_ENVIRONMENT=${venv_path}" >> "${GITHUB_ENV}"
         echo "UV_NO_SYNC=1" >> "${GITHUB_ENV}"

.github/workflows/gpu-tests.yml

@@ -105,31 +105,31 @@ jobs:
 
       - name: Run GPU unit tests
         timeout-minutes: 30
-        run: mise run test:unit:gpu
+        run: mise -E gpu-ci run test:unit:gpu
 
       - name: Run GPU smoke tests - train only
         timeout-minutes: 10
-        run: mise run test:smoke:gpu:train-only
+        run: mise -E gpu-ci run test:smoke:gpu:train-only
 
       - name: Run GPU smoke tests - generation
         timeout-minutes: 10
-        run: mise run test:smoke:gpu:generation
+        run: mise -E gpu-ci run test:smoke:gpu:generation
 
       - name: Run GPU smoke tests - resume
         timeout-minutes: 10
-        run: mise run test:smoke:gpu:resume
+        run: mise -E gpu-ci run test:smoke:gpu:resume
 
       - name: Run GPU smoke tests - structured generation
         timeout-minutes: 10
-        run: mise run test:smoke:gpu:structured-generation
+        run: mise -E gpu-ci run test:smoke:gpu:structured-generation
 
       - name: Run GPU smoke tests - timeseries
         timeout-minutes: 10
-        run: mise run test:smoke:gpu:timeseries
+        run: mise -E gpu-ci run test:smoke:gpu:timeseries
 
       - name: Run GPU smoke tests - SmolLM2
         timeout-minutes: 20
-        run: mise run test:smoke:gpu:smollm2
+        run: mise -E gpu-ci run test:smoke:gpu:smollm2
 
   gpu-e2e-test:
     name: GPU E2E Tests
@@ -162,7 +162,7 @@ jobs:
         uses: ./.github/actions/setup-gpu-test-env
 
       - name: Run GPU E2E tests
-        run: mise run test:e2e:prepared
+        run: mise -E gpu-ci run test:e2e:prepared
         timeout-minutes: 190
 
   # ---------------------------------------------------------------------------

.mise.gpu-ci.toml

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+[env]
+NSS_REQUIRE_CUDA = "1"

tests/TESTING.md

@@ -100,7 +100,7 @@ Defined in `pytest.ini` (`--strict-markers` is enabled):
 | `slow`         | Long-running tests                                                                               |
 | `smoke`        | Quick smoke tests (training/generation hot paths, tiny models)                                   |
 | `e2e`          | End-to-end pipeline tests (requires CUDA)                                                        |
-| `requires_gpu` | Test needs CUDA hardware (modifier, stacks on any category: `unit`/`smoke`/`e2e`)                |
+| `requires_gpu` | Test needs CUDA hardware; local runs skip it automatically when CUDA is unavailable              |
 | `vllm`         | Tests using vLLM generation backend (each file runs in its own process for GPU memory isolation) |
 | `smollm2`      | SmolLM2 Hub download tests (mise tasks use for process isolation)                                |
 | `noautouse`    | Skip autouse fixtures for specific tests                                                         |
@@ -118,6 +118,20 @@ The other markers modify the 3 categories, indicating when they should be run (`
 
 Markers are only added if none of the 3 category markers (`unit`, `smoke`, `e2e`) are already present on the test item.
 
+When at least one collected test has `requires_gpu`, the same hook checks `torch.cuda.is_available()`. In ordinary local runs,
+those tests are skipped automatically when CUDA is unavailable. PyTorch is not imported and CUDA is not probed when the
+collected tests do not use the marker.
+
+GPU CI uses the dedicated mise environment to turn missing PyTorch or unavailable CUDA into a collection error instead of
+silently skipping the GPU suite:
+
+```bash
+mise -E gpu-ci run test:smoke:gpu
+```
+
+The `gpu-ci` environment sets `NSS_REQUIRE_CUDA=1`. Use it only for lanes that promise GPU capability; generic CPU CI and
+ordinary local pytest runs must retain skip behavior.
+
 ## Test Data Locations
 
 | Location                                     | Contents                                                                                                                                                                                                                                                            |

tests/conftest.py

@@ -16,6 +16,8 @@
 import pytest
 from datasets import Dataset, load_dataset
 
+pytest_plugins = ("pytester",)
+
 
 def _pytest_live_logging_requested(config: pytest.Config) -> bool:
     """Return whether pytest is configured to stream test logs live."""
@@ -54,14 +56,18 @@ def pytest_configure(config: pytest.Config) -> None:
     initialize_observability()
 
 
-def pytest_collection_modifyitems(config, items):
-    """Auto-mark tests by directory: `/e2e/` -> e2e, `/smoke/` -> smoke, else unit."""
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Auto-mark test categories and skip GPU tests when CUDA is unavailable."""
     category_markers = {"unit", "e2e", "smoke"}
+    gpu_items = []
 
     for item in items:
         marker_names = {marker.name for marker in item.iter_markers()}
         path_str = str(item.fspath)
 
+        if "requires_gpu" in marker_names:
+            gpu_items.append(item)
+
         if "/e2e/" in path_str:
             if "e2e" not in marker_names:
                 item.add_marker(pytest.mark.e2e)
@@ -75,6 +81,30 @@ def pytest_collection_modifyitems(config, items):
         if not marker_names.intersection(category_markers):
             item.add_marker(pytest.mark.unit)
 
+    if not gpu_items:
+        return
+
+    require_cuda = os.environ.get("NSS_REQUIRE_CUDA") == "1"
+
+    try:
+        import torch
+    except ImportError:
+        if require_cuda:
+            raise pytest.UsageError(
+                "GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported."
+            ) from None
+        raise
+
+    if torch.cuda.is_available():
+        return
+
+    if require_cuda:
+        raise pytest.UsageError("GPU CI requires CUDA, but torch.cuda.is_available() returned false.")
+
+    skip_marker = pytest.mark.skip(reason="CUDA is not available")
+    for item in gpu_items:
+        item.add_marker(skip_marker)
+
 
 @pytest.fixture(autouse=True)
 def fixture_isolate_deployment_type(monkeypatch: pytest.MonkeyPatch) -> None:

tests/smoke/README.md

@@ -56,11 +56,13 @@ When adding a new GPU smoke test, add the appropriate markers to `pytestmark`:
 pytestmark = [
     pytest.mark.requires_gpu,
     pytest.mark.vllm,  # if the test calls .generate() (uses vLLM)
-    pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),
     pytest.mark.skipif(sys.platform == "darwin", reason="Not applicable on macOS"),
 ]
 ```
 
+The root pytest configuration automatically skips `requires_gpu` tests when CUDA is unavailable; do not add a per-test
+`torch.cuda.is_available()` skip condition.
+
 If the new file uses vLLM, add a dedicated `test:smoke:gpu:*` mise task and include it in `test:smoke:gpu` so CI shows it as its own stage.
 
 ## Things that will bite you