diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 53e314d50bb3..3739d66a5766 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1096,6 +1096,8 @@ def getPytestBaseCommandLine( if (stageName.contains("-Ray-")) { testCmdLine += ["--run-ray"] } + def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only" + testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"] if (extraArgs) { testCmdLine += extraArgs } @@ -2477,6 +2479,37 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod imagePullPolicy: Always""" nodeLabelPrefix = "cpu" break + case "cpu": + containerConfig = """ + - name: trt-llm + image: ${image} + command: ['sleep', ${POD_TIMEOUT_SECONDS_TEST}] + tty: true + resources: + requests: + cpu: ${TESTER_CORES} + memory: ${TESTER_MEMORY} + ephemeral-storage: 300Gi + limits: + cpu: ${TESTER_CORES} + memory: ${TESTER_MEMORY} + ephemeral-storage: 300Gi + imagePullPolicy: Always + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: scratch-trt-llm-data + mountPath: /scratch.trt_llm_data + readOnly: true + - name: sw-tensorrt-pvc + mountPath: "/mnt/sw-tensorrt-pvc" + readOnly: false + securityContext: + capabilities: + add: + - SYS_ADMIN""" + nodeLabelPrefix = "cpu" + break default: def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" @@ -2925,6 +2958,9 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { // If stageName contains "-AutoDeploy-", add "backend=autodeploy" to makoArgs // At this point, only tests with backend=autodeploy or unspecified backend will be run makoArgs += ["backend=autodeploy"] + } else if (stageName.contains("-Generic-")) { + // Generic stages select tests by marker expression rather than backend ownership. + makoArgs += ["backend=generic"] } else if (stageName.contains("-Verl-")) { // If stageName contains "-Verl-", add "backend=verl" to makoArgs // At this point, only tests with backend=verl or unspecified backend will be run @@ -2968,9 +3004,19 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu def makoOpts = preDefinedMakoOpts if (!makoOpts) { - def scriptPath = "${llmSrc}/tests/integration/defs/sysinfo/get_sysinfo.py" def makoArgs = getMakoArgsFromStageName(stageName) - makoOpts = getMakoOpts(scriptPath, makoArgs) + if (stageName.startsWith("CPU-")) { + def cpuName = env.targetArch == AARCH64_TRIPLE ? "aarch64" : "x86_64" + makoOpts = transformMakoArgsToJson( + ["Mako options:"] + makoArgs + [ + "system_gpu_count=0", + "cpu=${cpuName}", + "linux_distribution_name=ubuntu" + ]) + } else { + def scriptPath = "${llmSrc}/tests/integration/defs/sysinfo/get_sysinfo.py" + makoOpts = getMakoOpts(scriptPath, makoArgs) + } } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" @@ -3565,7 +3611,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO sh "nproc && free -g && hostname" echoNodeAndGpuInfo(pipeline, stageName) sh "cat ${MODEL_CACHE_DIR}/README" - sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m" + if (stageName.startsWith("CPU-")) { + sh "ln -s /usr/local/cuda/compat/lib.real /usr/local/cuda/compat/lib" + } else { + sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m" + } sh "df -h" // setup HF_HOME to cache model and datasets @@ -4533,6 +4583,7 @@ def launchTestJobs(pipeline, testFilter) // may break the mapping functionality. x86TestConfigs = [ + "CPU-Generic-x86-1": ["cpu", "l0_cpu_x86", 1, 1], "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], "A10-PyTorch-1": ["a10", "l0_a10", 1, 2], "A10-PyTorch-2": ["a10", "l0_a10", 2, 2], @@ -4731,6 +4782,7 @@ def launchTestJobs(pipeline, testFilter) // SBSA machines from the Blossom machine pool SBSATestConfigs = [ + "CPU-Generic-arm-1": ["cpu", "l0_cpu_arm", 1, 1], "GH200-TensorRT-Post-Merge-1": ["gh200", "l0_gh200", 1, 1], // DGX Spark is also named as GB10 Grace Blackwell Superchip. "GB10-PyTorch-1": ["gb10x", "l0_gb10", 1, 1], diff --git a/tensorrt_llm/_torch/attention_backend/flashinfer.py b/tensorrt_llm/_torch/attention_backend/flashinfer.py index 26fed0bf12bd..a19ad4c8a86f 100644 --- a/tensorrt_llm/_torch/attention_backend/flashinfer.py +++ b/tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -29,14 +29,14 @@ CustomAttentionMask, MLAParams, PredefinedAttentionMask, merge_attention_forward_args) -try: - check_cuda_arch() -except RuntimeError: - # Override TORCH_CUDA_ARCH_LIST for JIT compilation of flashinfer kernels - # since the existed TORCH_CUDA_ARCH_LIST may be too general and flashinfer requires sm75+. - # Guard on a visible GPU: with CUDA_VISIBLE_DEVICES="" (pure client) the - # capability query would force a CUDA context at import time. - if torch.cuda.is_available(): +# Guard on a visible GPU: with CUDA_VISIBLE_DEVICES="" (pure client) the +# check would force a CUDA context at import time. +if torch.cuda.is_available() and torch.cuda.device_count() > 0: + try: + check_cuda_arch() + except RuntimeError: + # Override TORCH_CUDA_ARCH_LIST for JIT compilation of flashinfer kernels + # since the existed TORCH_CUDA_ARCH_LIST may be too general and flashinfer requires sm75+. capability = torch.cuda.get_device_capability() arch_list = f"{capability[0]}.{capability[1]}" os.environ["TORCH_CUDA_ARCH_LIST"] = arch_list diff --git a/tensorrt_llm/_torch/auto_deploy/llm_args.py b/tensorrt_llm/_torch/auto_deploy/llm_args.py index 283e12ab8366..e1129641e5d6 100644 --- a/tensorrt_llm/_torch/auto_deploy/llm_args.py +++ b/tensorrt_llm/_torch/auto_deploy/llm_args.py @@ -16,7 +16,6 @@ from pathlib import Path from typing import Any, Dict, Literal, Optional, Type, Union -import torch from pydantic import Field, ValidationInfo, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -27,6 +26,7 @@ TorchLlmArgs, _ParallelConfig, ) +from tensorrt_llm.llmapi.utils import get_device_count from . import config as _ad_config_pkg from .models import ModelFactory, ModelFactoryRegistry @@ -89,7 +89,7 @@ class LlmArgs(DynamicYamlMixInForSettings, TorchLlmArgs, BaseSettings): ) gpus_per_node: int = Field( - default=torch.cuda.device_count(), + default=get_device_count(), description="The number of GPUs per node.", frozen=True, ) diff --git a/tensorrt_llm/_torch/cuda_tile_utils.py b/tensorrt_llm/_torch/cuda_tile_utils.py index 0ce76bc0117c..46336610a2d3 100644 --- a/tensorrt_llm/_torch/cuda_tile_utils.py +++ b/tensorrt_llm/_torch/cuda_tile_utils.py @@ -44,7 +44,7 @@ def ceil_div(a, b): return (a + b - 1) // b -if platform.system() != "Windows": +if platform.system() != "Windows" and torch.cuda.device_count() > 0: try: import cuda.tile # noqa: F401 except ImportError: diff --git a/tensorrt_llm/_utils.py b/tensorrt_llm/_utils.py index cf242cd205c9..cf7916446321 100644 --- a/tensorrt_llm/_utils.py +++ b/tensorrt_llm/_utils.py @@ -806,10 +806,16 @@ def release_gc(): torch.cuda.ipc_collect() -@lru_cache(maxsize=1) -def get_sm_version(): - prop = torch.cuda.get_device_properties(0) - return prop.major * 10 + prop.minor +if torch.cuda.device_count() == 0: + + def get_sm_version(): + return -1 +else: + + @lru_cache(maxsize=1) + def get_sm_version(): + prop = torch.cuda.get_device_properties(0) + return prop.major * 10 + prop.minor @lru_cache(maxsize=1) @@ -1379,7 +1385,7 @@ def prefer_pinned() -> bool: pageable (and not pinned) memory across the board is preferred in CC mode to maintain asynchronous execution. """ - return not confidential_compute_enabled() + return torch.cuda.device_count() > 0 and not confidential_compute_enabled() def maybe_pin_memory(tensor: torch.Tensor) -> torch.Tensor: diff --git a/tensorrt_llm/llmapi/utils.py b/tensorrt_llm/llmapi/utils.py index ffdc2e1096a3..f1b49fc87af8 100644 --- a/tensorrt_llm/llmapi/utils.py +++ b/tensorrt_llm/llmapi/utils.py @@ -149,7 +149,12 @@ def get_device_count() -> int: return torch.cuda.device_count() if torch.cuda.is_available() else 0 -def get_total_gpu_memory(device: int) -> float: +def get_total_gpu_memory(device: int) -> int: + # Compat for no GPU environment, only for device=0. + # Otherwise, the caller should ensure there are that many GPUs. + if device == 0 and get_device_count() == 0: + return 0 + return torch.cuda.get_device_properties(device).total_memory diff --git a/tensorrt_llm/profiler.py b/tensorrt_llm/profiler.py index 2ef29c845f2f..090657274191 100644 --- a/tensorrt_llm/profiler.py +++ b/tensorrt_llm/profiler.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import time from functools import partial from typing import Literal, Optional, Tuple, Union @@ -110,18 +111,24 @@ def summary(): MemUnitType = Literal["GiB", "MiB", "KiB"] -class PyNVMLContext: - def __enter__(self): - if pynvml is not None: +@contextlib.contextmanager +def pynvml_context(): + has_pynvml = pynvml is not None + if has_pynvml: + try: pynvml.nvmlInit() + except pynvml.NVMLError: + has_pynvml = False - def __exit__(self, type, value, traceback): - if pynvml is not None: + try: + yield + finally: + if has_pynvml: pynvml.nvmlShutdown() if pynvml is not None: - with PyNVMLContext(): + with pynvml_context(): _device_get_memory_info_fn = partial( pynvml.nvmlDeviceGetMemoryInfo, version=pynvml.nvmlMemory_v2, @@ -147,7 +154,7 @@ def device_memory_info(device: Optional[Union[torch.device, int]] = None) -> Tup if device is None: device = torch.cuda.current_device() index = device.index if isinstance(device, torch.device) else device - with PyNVMLContext(): + with pynvml_context(): handle = pynvml.nvmlDeviceGetHandleByIndex(index) mem_info = _device_get_memory_info_fn(handle) return mem_info.used, mem_info.free, mem_info.total diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index cb8fc5bac3fa..bba88cf07c01 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -1807,13 +1807,17 @@ def get_gpu_device_list(): suffix = ".exe" if is_windows() else "" # TODO: Use NRSU because we can't assume nvidia-smi across all platforms. cmd = " ".join(["nvidia-smi" + suffix, "-L"]) - output = check_output(cmd, shell=True, cwd=temp_dirname) + try: + output = check_output(cmd, shell=True, cwd=temp_dirname) + except sp.CalledProcessError: + return [] return [l.strip() for l in output.strip().split("\n")] def check_device_contain(keyword_list): "check device not contain keyword" - device = get_gpu_device_list()[0] + devices = get_gpu_device_list() + device = devices[0] if devices else "" return any(keyword in device for keyword in keyword_list) @@ -2008,6 +2012,12 @@ def pytest_addoption(parser): help= "Enable Ray orchestrator path for integration tests (disables MPI).", ) + parser.addoption( + "--unittest-markexpr", + action="store", + default=None, + help="Marker expression forwarded to nested unittest pytest runs.", + ) parser.addoption( "--perf-log-formats", help= diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index a535044e69af..9287fe9b3ca5 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -66,7 +66,12 @@ def merge_attr(name, type_=int): def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): import pandas as pd import pynvml - pynvml.nvmlInit() + try: + pynvml.nvmlInit() + gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0) + gpu_name = pynvml.nvmlDeviceGetName(gpu_handle) + except pynvml.NVMLError: + gpu_name = "CPU" test_root = tests_path() dry_run = False @@ -79,6 +84,7 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): waives_file = request.config.getoption("--waives-file") run_ray = request.config.getoption("--run-ray") + unittest_markexpr = request.config.getoption("--unittest-markexpr") num_workers = 1 @@ -87,8 +93,6 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): agg_unit_mem_path = f'{test_root}/integration/defs/agg_unit_mem_df.csv' print(f'Loading unittest parallel config from: {agg_unit_mem_path}') agg_unit_mem_df = pd.read_csv(agg_unit_mem_path) - gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0) - gpu_name = pynvml.nvmlDeviceGetName(gpu_handle) if isinstance(gpu_name, bytes): gpu_name = gpu_name.decode() print(f'GPU name from NVML (index 0): {gpu_name!r}') @@ -167,6 +171,9 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): if run_ray: command += ["--run-ray"] + if unittest_markexpr: + command += ["-m", unittest_markexpr] + s3_secret_key = None s3_upload_path = request.config.getoption("--s3-upload-path", default=None) if s3_upload_path: diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 1e6836fc7cd5..678adad9a4ce 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -139,7 +139,6 @@ l0_a10: - unittest/llmapi/test_additional_model_outputs.py -m "gpu1" - unittest/llmapi/test_request_priority.py # executor - - unittest/executor/test_rpc.py - unittest/executor/test_ipc.py - unittest/executor/test_fatal_error_health_check.py - unittest/executor/test_postprocessor_hook.py diff --git a/tests/integration/test_lists/test-db/l0_cpu_arm.yml b/tests/integration/test_lists/test-db/l0_cpu_arm.yml new file mode 100644 index 000000000000..ba693acaf756 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_cpu_arm.yml @@ -0,0 +1,16 @@ +version: 0.0.1 +l0_cpu_arm: +- condition: + ranges: + system_gpu_count: + gte: 0 + lte: 0 + wildcards: + linux_distribution_name: ubuntu* + cpu: aarch64 + terms: + stage: pre_merge + backend: generic + orchestrator: mpi + tests: + - unittest/executor/test_rpc.py diff --git a/tests/integration/test_lists/test-db/l0_cpu_x86.yml b/tests/integration/test_lists/test-db/l0_cpu_x86.yml new file mode 100644 index 000000000000..9a39993347b8 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_cpu_x86.yml @@ -0,0 +1,16 @@ +version: 0.0.1 +l0_cpu_x86: +- condition: + ranges: + system_gpu_count: + gte: 0 + lte: 0 + wildcards: + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: pre_merge + backend: generic + orchestrator: mpi + tests: + - unittest/executor/test_rpc.py diff --git a/tests/unittest/auto_deploy/_utils_test/_torch_test_utils.py b/tests/unittest/auto_deploy/_utils_test/_torch_test_utils.py index 447f6733e949..a9fb048f063c 100644 --- a/tests/unittest/auto_deploy/_utils_test/_torch_test_utils.py +++ b/tests/unittest/auto_deploy/_utils_test/_torch_test_utils.py @@ -41,10 +41,14 @@ def reset_parameters(model: torch.nn.Module): def fp8_compatible(): + if torch.cuda.device_count() == 0: + return False return torch.cuda.get_device_capability(0) >= (8, 9) def fp4_compatible(): + if torch.cuda.device_count() == 0: + return False return torch.cuda.get_device_capability(0) >= (10, 0) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index 15901385a255..57bd556dbd14 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -19,6 +19,7 @@ import traceback import warnings from functools import partial +from pathlib import Path from typing import Any, Generator try: @@ -106,6 +107,9 @@ def pytest_runtest_protocol(item, nextitem): import os import torch + if torch.cuda.device_count() == 0: + return + worker_count = int(os.environ.get('PYTEST_XDIST_WORKER_COUNT', 1)) if (torch.cuda.memory_reserved(0) + torch.cuda.memory_allocated(0) @@ -209,6 +213,27 @@ def pytest_addoption(parser): s3_output.add_options(parser) +def _is_cpu_only_markexpr(config) -> bool: + markexpr = getattr(config.option, "markexpr", "") or "" + return "cpu_only" in markexpr and "not cpu_only" not in markexpr + + +def pytest_ignore_collect(collection_path, config): + if not _is_cpu_only_markexpr(config): + return None + + path = Path(str(collection_path)) + if path.name == "conftest.py" or path.suffix != ".py": + return None + if not (path.name.startswith("test_") or path.name.endswith("_test.py")): + return None + + try: + return "pytest.mark.cpu_only" not in path.read_text() + except OSError: + return None + + def apply_waives_ut(waives_file, items: list[pytest.Item], config): """Apply waives based on the waive state specified by the given waives_file.""" diff --git a/tests/unittest/executor/test_rpc.py b/tests/unittest/executor/test_rpc.py index 91a444ecc9cf..63042ef85ef7 100644 --- a/tests/unittest/executor/test_rpc.py +++ b/tests/unittest/executor/test_rpc.py @@ -9,6 +9,8 @@ RPCServer, RPCStreamingError, RPCTimeout) from tensorrt_llm.executor.rpc.rpc_common import get_unique_ipc_addr +pytestmark = pytest.mark.cpu_only + class RpcServerWrapper(RPCServer): """ A helper class to wrap the RPCServer and manage its lifecycle. """ diff --git a/tests/unittest/pytest.ini b/tests/unittest/pytest.ini index 62e38297239d..b0f947f294c8 100644 --- a/tests/unittest/pytest.ini +++ b/tests/unittest/pytest.ini @@ -19,6 +19,7 @@ markers = part2 part3 part4 + cpu_only: this test does not require a GPU to run gpu2: this test uses 2 GPUs gpu4: this test uses 4 GPUs post_merge: this test should only run in post merge diff --git a/tests/unittest/utils/util.py b/tests/unittest/utils/util.py index 4a2b2bb435d2..fbb2808acb6a 100644 --- a/tests/unittest/utils/util.py +++ b/tests/unittest/utils/util.py @@ -63,6 +63,8 @@ def ASSERT_DRV(err): def getSMVersion(): # Init err, = cuda.cuInit(0) + if err == cuda.CUresult.CUDA_ERROR_NO_DEVICE: + return -1 ASSERT_DRV(err) # Device