diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 7c4c41769..d1d99125d 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -84,6 +84,7 @@ runs: - '.github/workflows/**' - '.github/actions/**' - '.mise.toml' + - '.mise.*.toml' - '.mise/tasks/**' - '.dockerignore' - 'containers/Dockerfile.test_ci' diff --git a/.github/actions/setup-gpu-test-env/action.yml b/.github/actions/setup-gpu-test-env/action.yml index 1becddc7a..8778d9371 100644 --- a/.github/actions/setup-gpu-test-env/action.yml +++ b/.github/actions/setup-gpu-test-env/action.yml @@ -118,8 +118,18 @@ runs: uv --no-config pip check --python "${python_bin}" "${python_bin}" -c \ "from importlib.metadata import version; from nemo_safe_synthesizer.package_info import __version__; assert __version__ == version('nemo-safe-synthesizer')" - "${python_bin}" -c \ - "import torch; print('cuda available:', torch.cuda.is_available()); print('device count:', torch.cuda.device_count()); assert torch.cuda.is_available()" + "${python_bin}" - <<'PY' + try: + import torch + except ImportError: + raise SystemExit("GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported.") from None + + if not torch.cuda.is_available(): + raise SystemExit("GPU CI requires CUDA, but torch.cuda.is_available() returned false.") + + print("cuda available: true") + print("device count:", torch.cuda.device_count()) + PY echo "UV_PROJECT_ENVIRONMENT=${venv_path}" >> "${GITHUB_ENV}" echo "UV_NO_SYNC=1" >> "${GITHUB_ENV}" diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 6f9e379ae..a4c644b2f 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -105,31 +105,31 @@ jobs: - name: Run GPU unit tests timeout-minutes: 30 - run: mise run test:unit:gpu + run: mise -E gpu-ci run test:unit:gpu - name: Run GPU smoke tests - train only timeout-minutes: 10 - run: mise run test:smoke:gpu:train-only + run: mise -E gpu-ci run test:smoke:gpu:train-only - name: Run GPU smoke tests - generation timeout-minutes: 10 - run: mise run test:smoke:gpu:generation + run: mise -E gpu-ci run test:smoke:gpu:generation - name: Run GPU smoke tests - resume timeout-minutes: 10 - run: mise run test:smoke:gpu:resume + run: mise -E gpu-ci run test:smoke:gpu:resume - name: Run GPU smoke tests - structured generation timeout-minutes: 10 - run: mise run test:smoke:gpu:structured-generation + run: mise -E gpu-ci run test:smoke:gpu:structured-generation - name: Run GPU smoke tests - timeseries timeout-minutes: 10 - run: mise run test:smoke:gpu:timeseries + run: mise -E gpu-ci run test:smoke:gpu:timeseries - name: Run GPU smoke tests - SmolLM2 timeout-minutes: 20 - run: mise run test:smoke:gpu:smollm2 + run: mise -E gpu-ci run test:smoke:gpu:smollm2 gpu-e2e-test: name: GPU E2E Tests @@ -162,7 +162,7 @@ jobs: uses: ./.github/actions/setup-gpu-test-env - name: Run GPU E2E tests - run: mise run test:e2e:prepared + run: mise -E gpu-ci run test:e2e:prepared timeout-minutes: 190 # --------------------------------------------------------------------------- diff --git a/.mise.gpu-ci.toml b/.mise.gpu-ci.toml new file mode 100644 index 000000000..50fbe37f5 --- /dev/null +++ b/.mise.gpu-ci.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[env] +NSS_REQUIRE_CUDA = "1" diff --git a/tests/TESTING.md b/tests/TESTING.md index fc2231d1a..8cf02445d 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -100,7 +100,7 @@ Defined in `pytest.ini` (`--strict-markers` is enabled): | `slow` | Long-running tests | | `smoke` | Quick smoke tests (training/generation hot paths, tiny models) | | `e2e` | End-to-end pipeline tests (requires CUDA) | -| `requires_gpu` | Test needs CUDA hardware (modifier, stacks on any category: `unit`/`smoke`/`e2e`) | +| `requires_gpu` | Test needs CUDA hardware; local runs skip it automatically when CUDA is unavailable | | `vllm` | Tests using vLLM generation backend (each file runs in its own process for GPU memory isolation) | | `smollm2` | SmolLM2 Hub download tests (mise tasks use for process isolation) | | `noautouse` | Skip autouse fixtures for specific tests | @@ -118,6 +118,20 @@ The other markers modify the 3 categories, indicating when they should be run (` Markers are only added if none of the 3 category markers (`unit`, `smoke`, `e2e`) are already present on the test item. +When at least one collected test has `requires_gpu`, the same hook checks `torch.cuda.is_available()`. In ordinary local runs, +those tests are skipped automatically when CUDA is unavailable. PyTorch is not imported and CUDA is not probed when the +collected tests do not use the marker. + +GPU CI uses the dedicated mise environment to turn missing PyTorch or unavailable CUDA into a collection error instead of +silently skipping the GPU suite: + +```bash +mise -E gpu-ci run test:smoke:gpu +``` + +The `gpu-ci` environment sets `NSS_REQUIRE_CUDA=1`. Use it only for lanes that promise GPU capability; generic CPU CI and +ordinary local pytest runs must retain skip behavior. + ## Test Data Locations | Location | Contents | diff --git a/tests/conftest.py b/tests/conftest.py index ee89adfe4..5fd840d44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,8 @@ import pytest from datasets import Dataset, load_dataset +pytest_plugins = ("pytester",) + def _pytest_live_logging_requested(config: pytest.Config) -> bool: """Return whether pytest is configured to stream test logs live.""" @@ -54,14 +56,18 @@ def pytest_configure(config: pytest.Config) -> None: initialize_observability() -def pytest_collection_modifyitems(config, items): - """Auto-mark tests by directory: `/e2e/` -> e2e, `/smoke/` -> smoke, else unit.""" +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Auto-mark test categories and skip GPU tests when CUDA is unavailable.""" category_markers = {"unit", "e2e", "smoke"} + gpu_items = [] for item in items: marker_names = {marker.name for marker in item.iter_markers()} path_str = str(item.fspath) + if "requires_gpu" in marker_names: + gpu_items.append(item) + if "/e2e/" in path_str: if "e2e" not in marker_names: item.add_marker(pytest.mark.e2e) @@ -75,6 +81,30 @@ def pytest_collection_modifyitems(config, items): if not marker_names.intersection(category_markers): item.add_marker(pytest.mark.unit) + if not gpu_items: + return + + require_cuda = os.environ.get("NSS_REQUIRE_CUDA") == "1" + + try: + import torch + except ImportError: + if require_cuda: + raise pytest.UsageError( + "GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported." + ) from None + raise + + if torch.cuda.is_available(): + return + + if require_cuda: + raise pytest.UsageError("GPU CI requires CUDA, but torch.cuda.is_available() returned false.") + + skip_marker = pytest.mark.skip(reason="CUDA is not available") + for item in gpu_items: + item.add_marker(skip_marker) + @pytest.fixture(autouse=True) def fixture_isolate_deployment_type(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/generation/test_vllm_shutdown.py b/tests/generation/test_vllm_shutdown.py index a7d88d850..6285dacf1 100644 --- a/tests/generation/test_vllm_shutdown.py +++ b/tests/generation/test_vllm_shutdown.py @@ -52,6 +52,8 @@ class TestTeardownIdempotency: def test_first_teardown_runs_cleanup(self, backend, _mock_vllm_cleanup): mock_dist, mock_mem = _mock_vllm_cleanup backend.llm = MagicMock() + mock_dist.reset_mock() + mock_mem.reset_mock() backend.teardown() diff --git a/tests/smoke/README.md b/tests/smoke/README.md index 13fdacac0..5a9fe46a8 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -56,11 +56,13 @@ When adding a new GPU smoke test, add the appropriate markers to `pytestmark`: pytestmark = [ pytest.mark.requires_gpu, pytest.mark.vllm, # if the test calls .generate() (uses vLLM) - pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"), pytest.mark.skipif(sys.platform == "darwin", reason="Not applicable on macOS"), ] ``` +The root pytest configuration automatically skips `requires_gpu` tests when CUDA is unavailable; do not add a per-test +`torch.cuda.is_available()` skip condition. + If the new file uses vLLM, add a dedicated `test:smoke:gpu:*` mise task and include it in `test:smoke:gpu` so CI shows it as its own stage. ## Things that will bite you diff --git a/tests/test_pytest_configuration.py b/tests/test_pytest_configuration.py new file mode 100644 index 000000000..08820cb00 --- /dev/null +++ b/tests/test_pytest_configuration.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for repository-wide pytest configuration.""" + +from __future__ import annotations + +import builtins +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +def _root_conftest_module(config: pytest.Config) -> ModuleType: + """Return the loaded root test configuration module.""" + expected_path = (config.rootpath / "tests" / "conftest.py").resolve() + for plugin in config.pluginmanager.get_plugins(): + plugin_path = getattr(plugin, "__file__", None) + if plugin_path is not None and Path(plugin_path).resolve() == expected_path: + assert isinstance(plugin, ModuleType) + return plugin + raise AssertionError(f"Root conftest plugin not found at {expected_path}") + + +def _make_pytest_ini(pytester: pytest.Pytester) -> None: + """Register the GPU marker in pytester's isolated test environment.""" + pytester.makeini( + """ + [pytest] + markers = + requires_gpu: Test needs CUDA hardware + """ + ) + + +def test_requires_gpu_is_skipped_without_cuda( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=False) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + test_gpu=""" + import pytest + + pytestmark = pytest.mark.requires_gpu + + def test_module_marked_gpu(): + raise AssertionError("GPU test ran without CUDA") + + @pytest.mark.requires_gpu + def test_function_marked_gpu(): + raise AssertionError("GPU test ran without CUDA") + """, + test_cpu=""" + def test_cpu(): + pass + """, + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1, skipped=2) + cuda_is_available.assert_called_once_with() + + +def test_requires_gpu_runs_with_cuda( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=True) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1) + cuda_is_available.assert_called_once_with() + + +def test_requires_gpu_fails_without_torch_in_gpu_ci( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") + monkeypatch.setitem(sys.modules, "torch", None) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*GPU CI requires PyTorch with CUDA support, but PyTorch could not be imported.*"]) + + +def test_requires_gpu_fails_without_cuda_in_gpu_ci( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=False) + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*GPU CI requires CUDA, but torch.cuda.is_available() returned false.*"]) + cuda_is_available.assert_called_once_with() + + +def test_requires_gpu_runs_with_cuda_in_gpu_ci( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + cuda_is_available = Mock(return_value=True) + monkeypatch.setenv("NSS_REQUIRE_CUDA", "1") + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=cuda_is_available))) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.requires_gpu + def test_gpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1) + cuda_is_available.assert_called_once_with() + + +def test_torch_is_not_imported_without_gpu_tests( + pytester: pytest.Pytester, + pytestconfig: pytest.Config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_import = builtins.__import__ + + def fail_on_torch_import( + name: str, + globals_: Mapping[str, object] | None = None, + locals_: Mapping[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, + ) -> ModuleType: + if name == "torch": + raise AssertionError("Torch was imported without a collected requires_gpu test") + return real_import(name, globals_, locals_, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_on_torch_import) + _make_pytest_ini(pytester) + pytester.makepyfile( + """ + def test_cpu(): + pass + """ + ) + + result = pytester.runpytest_inprocess( + "-p", + "no:cov", + plugins=[_root_conftest_module(pytestconfig)], + ) + + result.assert_outcomes(passed=1)