From fa5e7044dc77b46c93668e42261e26535e5eef7a Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 28 Jul 2026 17:55:00 +0000 Subject: [PATCH 1/4] test: skip GPU tests when CUDA is unavailable Signed-off-by: Aaron Gonzales --- tests/TESTING.md | 6 +- tests/conftest.py | 22 ++++++- tests/smoke/README.md | 4 +- tests/test_pytest_configuration.py | 97 ++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 tests/test_pytest_configuration.py diff --git a/tests/TESTING.md b/tests/TESTING.md index fc2231d1a..82df3b31d 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -100,7 +100,7 @@ Defined in `pytest.ini` (`--strict-markers` is enabled): | `slow` | Long-running tests | | `smoke` | Quick smoke tests (training/generation hot paths, tiny models) | | `e2e` | End-to-end pipeline tests (requires CUDA) | -| `requires_gpu` | Test needs CUDA hardware (modifier, stacks on any category: `unit`/`smoke`/`e2e`) | +| `requires_gpu` | Test needs CUDA hardware and is automatically skipped when CUDA is unavailable | | `vllm` | Tests using vLLM generation backend (each file runs in its own process for GPU memory isolation) | | `smollm2` | SmolLM2 Hub download tests (mise tasks use for process isolation) | | `noautouse` | Skip autouse fixtures for specific tests | @@ -118,6 +118,10 @@ The other markers modify the 3 categories, indicating when they should be run (` Markers are only added if none of the 3 category markers (`unit`, `smoke`, `e2e`) are already present on the test item. +When at least one collected test has `requires_gpu`, the same hook checks `torch.cuda.is_available()`. If CUDA is unavailable, +those tests are skipped automatically. PyTorch is not imported and CUDA is not probed when the collected tests do not use the +marker. + ## Test Data Locations | Location | Contents | diff --git a/tests/conftest.py b/tests/conftest.py index ee89adfe4..577a1734a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,8 @@ import pytest from datasets import Dataset, load_dataset +pytest_plugins = ("pytester",) + def _pytest_live_logging_requested(config: pytest.Config) -> bool: """Return whether pytest is configured to stream test logs live.""" @@ -54,14 +56,18 @@ def pytest_configure(config: pytest.Config) -> None: initialize_observability() -def pytest_collection_modifyitems(config, items): - """Auto-mark tests by directory: `/e2e/` -> e2e, `/smoke/` -> smoke, else unit.""" +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Auto-mark test categories and skip GPU tests when CUDA is unavailable.""" category_markers = {"unit", "e2e", "smoke"} + gpu_items = [] for item in items: marker_names = {marker.name for marker in item.iter_markers()} path_str = str(item.fspath) + if "requires_gpu" in marker_names: + gpu_items.append(item) + if "/e2e/" in path_str: if "e2e" not in marker_names: item.add_marker(pytest.mark.e2e) @@ -75,6 +81,18 @@ def pytest_collection_modifyitems(config, items): if not marker_names.intersection(category_markers): item.add_marker(pytest.mark.unit) + if not gpu_items: + return + + import torch + + if torch.cuda.is_available(): + return + + skip_marker = pytest.mark.skip(reason="CUDA is not available") + for item in gpu_items: + item.add_marker(skip_marker) + @pytest.fixture(autouse=True) def fixture_isolate_deployment_type(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/smoke/README.md b/tests/smoke/README.md index 13fdacac0..5a9fe46a8 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -56,11 +56,13 @@ When adding a new GPU smoke test, add the appropriate markers to `pytestmark`: pytestmark = [ pytest.mark.requires_gpu, pytest.mark.vllm, # if the test calls .generate() (uses vLLM) - pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"), pytest.mark.skipif(sys.platform == "darwin", reason="Not applicable on macOS"), ] ``` +The root pytest configuration automatically skips `requires_gpu` tests when CUDA is unavailable; do not add a per-test +`torch.cuda.is_available()` skip condition. + If the new file uses vLLM, add a dedicated `test:smoke:gpu:*` mise task and include it in `test:smoke:gpu` so CI shows it as its own stage. ## Things that will bite you diff --git a/tests/test_pytest_configuration.py b/tests/test_pytest_configuration.py new file mode 100644 index 000000000..3b5b96a79 --- /dev/null +++ b/tests/test_pytest_configuration.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for repository-wide pytest configuration.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +def _root_conftest_module(config: pytest.Config) -> ModuleType: + """Return the loaded root test configuration module.""" + expected_path = (config.rootpath / "tests" / "conftest.py").resolve() + for plugin in config.pluginmanager.get_plugins(): + plugin_path = getattr(plugin, "__file__", None) + if plugin_path is not None and Path(plugin_path).resolve() == expected_path: + assert isinstance(plugin, ModuleType) + return plugin + raise AssertionError(f"Root conftest plugin not found at {expected_path}") + + +def _make_pytest_ini(pytester: pytest.Pytester) -> None: + """Register the GPU marker in pytester's isolated test environment.""" + pytester.makeini( + """ + [pytest] + markers = + requires_gpu: Test needs CUDA hardware + """ + ) + + +def test_requires_gpu_is_skipped_without_cuda( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=False) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + test_gpu=""" + import pytest + + pytestmark = pytest.mark.requires_gpu + + def test_module_marked_gpu(): + raise AssertionError("GPU test ran without CUDA") + + @pytest.mark.requires_gpu + def test_function_marked_gpu(): + raise AssertionError("GPU test ran without CUDA") + """, + test_cpu=""" + def test_cpu(): + pass + """, + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1, skipped=2) + cuda_is_available.assert_called_once_with() + + +def test_cuda_is_not_probed_without_gpu_tests( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=False) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + def test_cpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1) + cuda_is_available.assert_not_called() From da3eb5170db9eb720e8223e2296fcc87fa3da4ff Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 29 Jul 2026 17:25:33 +0000 Subject: [PATCH 2/4] test: cover CUDA-available collection path Signed-off-by: Aaron Gonzales --- tests/test_pytest_configuration.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_pytest_configuration.py b/tests/test_pytest_configuration.py index 3b5b96a79..2ec8a22fe 100644 --- a/tests/test_pytest_configuration.py +++ b/tests/test_pytest_configuration.py @@ -72,6 +72,34 @@ def test_cpu(): cuda_is_available.assert_called_once_with() +def test_requires_gpu_runs_with_cuda( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=True) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1) + cuda_is_available.assert_called_once_with() + + def test_cuda_is_not_probed_without_gpu_tests( pytester: pytest.Pytester, pytestconfig: pytest.Config, From 06fab775b422cfd477ae2e9d5f6f4b0d6bccdf82 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 5 Aug 2026 18:11:24 +0000 Subject: [PATCH 3/4] test: require CUDA for GPU CI Signed-off-by: Aaron Gonzales --- .github/actions/detect-changes/action.yml | 1 + .github/actions/setup-gpu-test-env/action.yml | 14 ++- .github/workflows/gpu-tests.yml | 16 +-- .mise.gpu-ci.toml | 5 + tests/TESTING.md | 18 ++- tests/conftest.py | 14 ++- tests/test_pytest_configuration.py | 104 +++++++++++++++++- 7 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 .mise.gpu-ci.toml diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 7c4c41769..d1d99125d 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -84,6 +84,7 @@ runs: - '.github/workflows/**' - '.github/actions/**' - '.mise.toml' + - '.mise.*.toml' - '.mise/tasks/**' - '.dockerignore' - 'containers/Dockerfile.test_ci' diff --git a/.github/actions/setup-gpu-test-env/action.yml b/.github/actions/setup-gpu-test-env/action.yml index 1becddc7a..8778d9371 100644 --- a/.github/actions/setup-gpu-test-env/action.yml +++ b/.github/actions/setup-gpu-test-env/action.yml @@ -118,8 +118,18 @@ runs: uv --no-config pip check --python "${python_bin}" "${python_bin}" -c \ "from importlib.metadata import version; from nemo_safe_synthesizer.package_info import __version__; assert __version__ == version('nemo-safe-synthesizer')" - "${python_bin}" -c \ - "import torch; print('cuda available:', torch.cuda.is_available()); print('device count:', torch.cuda.device_count()); assert torch.cuda.is_available()" + "${python_bin}" - <<'PY' + try: + import torch + except ImportError: + raise SystemExit("GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported.") from None + + if not torch.cuda.is_available(): + raise SystemExit("GPU CI requires CUDA, but torch.cuda.is_available() returned false.") + + print("cuda available: true") + print("device count:", torch.cuda.device_count()) + PY echo "UV_PROJECT_ENVIRONMENT=${venv_path}" >> "${GITHUB_ENV}" echo "UV_NO_SYNC=1" >> "${GITHUB_ENV}" diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 6f9e379ae..a4c644b2f 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -105,31 +105,31 @@ jobs: - name: Run GPU unit tests timeout-minutes: 30 - run: mise run test:unit:gpu + run: mise -E gpu-ci run test:unit:gpu - name: Run GPU smoke tests - train only timeout-minutes: 10 - run: mise run test:smoke:gpu:train-only + run: mise -E gpu-ci run test:smoke:gpu:train-only - name: Run GPU smoke tests - generation timeout-minutes: 10 - run: mise run test:smoke:gpu:generation + run: mise -E gpu-ci run test:smoke:gpu:generation - name: Run GPU smoke tests - resume timeout-minutes: 10 - run: mise run test:smoke:gpu:resume + run: mise -E gpu-ci run test:smoke:gpu:resume - name: Run GPU smoke tests - structured generation timeout-minutes: 10 - run: mise run test:smoke:gpu:structured-generation + run: mise -E gpu-ci run test:smoke:gpu:structured-generation - name: Run GPU smoke tests - timeseries timeout-minutes: 10 - run: mise run test:smoke:gpu:timeseries + run: mise -E gpu-ci run test:smoke:gpu:timeseries - name: Run GPU smoke tests - SmolLM2 timeout-minutes: 20 - run: mise run test:smoke:gpu:smollm2 + run: mise -E gpu-ci run test:smoke:gpu:smollm2 gpu-e2e-test: name: GPU E2E Tests @@ -162,7 +162,7 @@ jobs: uses: ./.github/actions/setup-gpu-test-env - name: Run GPU E2E tests - run: mise run test:e2e:prepared + run: mise -E gpu-ci run test:e2e:prepared timeout-minutes: 190 # --------------------------------------------------------------------------- diff --git a/.mise.gpu-ci.toml b/.mise.gpu-ci.toml new file mode 100644 index 000000000..50fbe37f5 --- /dev/null +++ b/.mise.gpu-ci.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[env] +NSS_REQUIRE_CUDA = "1" diff --git a/tests/TESTING.md b/tests/TESTING.md index 82df3b31d..8cf02445d 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -100,7 +100,7 @@ Defined in `pytest.ini` (`--strict-markers` is enabled): | `slow` | Long-running tests | | `smoke` | Quick smoke tests (training/generation hot paths, tiny models) | | `e2e` | End-to-end pipeline tests (requires CUDA) | -| `requires_gpu` | Test needs CUDA hardware and is automatically skipped when CUDA is unavailable | +| `requires_gpu` | Test needs CUDA hardware; local runs skip it automatically when CUDA is unavailable | | `vllm` | Tests using vLLM generation backend (each file runs in its own process for GPU memory isolation) | | `smollm2` | SmolLM2 Hub download tests (mise tasks use for process isolation) | | `noautouse` | Skip autouse fixtures for specific tests | @@ -118,9 +118,19 @@ The other markers modify the 3 categories, indicating when they should be run (` Markers are only added if none of the 3 category markers (`unit`, `smoke`, `e2e`) are already present on the test item. -When at least one collected test has `requires_gpu`, the same hook checks `torch.cuda.is_available()`. If CUDA is unavailable, -those tests are skipped automatically. PyTorch is not imported and CUDA is not probed when the collected tests do not use the -marker. +When at least one collected test has `requires_gpu`, the same hook checks `torch.cuda.is_available()`. In ordinary local runs, +those tests are skipped automatically when CUDA is unavailable. PyTorch is not imported and CUDA is not probed when the +collected tests do not use the marker. + +GPU CI uses the dedicated mise environment to turn missing PyTorch or unavailable CUDA into a collection error instead of +silently skipping the GPU suite: + +```bash +mise -E gpu-ci run test:smoke:gpu +``` + +The `gpu-ci` environment sets `NSS_REQUIRE_CUDA=1`. Use it only for lanes that promise GPU capability; generic CPU CI and +ordinary local pytest runs must retain skip behavior. ## Test Data Locations diff --git a/tests/conftest.py b/tests/conftest.py index 577a1734a..5fd840d44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,11 +84,23 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item if not gpu_items: return - import torch + require_cuda = os.environ.get("NSS_REQUIRE_CUDA") == "1" + + try: + import torch + except ImportError: + if require_cuda: + raise pytest.UsageError( + "GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported." + ) from None + raise if torch.cuda.is_available(): return + if require_cuda: + raise pytest.UsageError("GPU CI requires CUDA, but torch.cuda.is_available() returned false.") + skip_marker = pytest.mark.skip(reason="CUDA is not available") for item in gpu_items: item.add_marker(skip_marker) diff --git a/tests/test_pytest_configuration.py b/tests/test_pytest_configuration.py index 2ec8a22fe..08820cb00 100644 --- a/tests/test_pytest_configuration.py +++ b/tests/test_pytest_configuration.py @@ -5,7 +5,9 @@ from __future__ import annotations +import builtins import sys +from collections.abc import Mapping, Sequence from pathlib import Path from types import ModuleType, SimpleNamespace from unittest.mock import Mock @@ -100,14 +102,113 @@ def test_gpu(): cuda_is_available.assert_called_once_with() -def test_cuda_is_not_probed_without_gpu_tests( +def test_requires_gpu_fails_without_torch_in_gpu_ci( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") + monkeypatch.setitem(sys.modules, "torch", None) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported.*"]) + + +def test_requires_gpu_fails_without_cuda_in_gpu_ci( pytester: pytest.Pytester, pytestconfig: pytest.Config, monkeypatch: pytest.MonkeyPatch, ) -> None: cuda_is_available = Mock(return_value=False) + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*GPU CI requires CUDA, but torch.cuda.is_available() returned false.*"]) + cuda_is_available.assert_called_once_with() + + +def test_requires_gpu_runs_with_cuda_in_gpu_ci( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=True) + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1) + cuda_is_available.assert_called_once_with() + + +def test_torch_is_not_imported_without_gpu_tests( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_import = builtins.__import__ + + def fail_on_torch_import( + name: str, + globals_: Mapping[str, object] | None = None, + locals_: Mapping[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, + ) -> ModuleType: + if name == "torch": + raise AssertionError("Torch was imported without a collected requires_gpu test") + return real_import(name, globals_, locals_, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_on_torch_import) + _make_pytest_ini(pytester) pytester.makepyfile( """ def test_cpu(): @@ -122,4 +223,3 @@ def test_cpu(): ) result.assert_outcomes(passed=1) - cuda_is_available.assert_not_called() From 00cc334f2994ece1bb43a9b6c455e03943103533 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 5 Aug 2026 23:26:27 +0000 Subject: [PATCH 4/4] test: isolate vLLM teardown cleanup assertions Signed-off-by: Aaron Gonzales --- tests/generation/test_vllm_shutdown.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/generation/test_vllm_shutdown.py b/tests/generation/test_vllm_shutdown.py index a7d88d850..6285dacf1 100644 --- a/tests/generation/test_vllm_shutdown.py +++ b/tests/generation/test_vllm_shutdown.py @@ -52,6 +52,8 @@ class TestTeardownIdempotency: def test_first_teardown_runs_cleanup(self, backend, _mock_vllm_cleanup): mock_dist, mock_mem = _mock_vllm_cleanup backend.llm = MagicMock() + mock_dist.reset_mock() + mock_mem.reset_mock() backend.teardown()