[TRTLLM-13696][test] Part1: Add CPU only CI stage

jenkins/L0_Test.groovy

@@ -1096,6 +1096,8 @@ def getPytestBaseCommandLine(
     if (stageName.contains("-Ray-")) {
         testCmdLine += ["--run-ray"]
     }
+    def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only"
+    testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"]
     if (extraArgs) {
         testCmdLine += extraArgs
     }
@@ -2477,6 +2479,37 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod
                     imagePullPolicy: Always"""
         nodeLabelPrefix = "cpu"
         break
+    case "cpu":
+        containerConfig = """
+                  - name: trt-llm
+                    image: ${image}
+                    command: ['sleep', ${POD_TIMEOUT_SECONDS_TEST}]
+                    tty: true
+                    resources:
+                      requests:
+                        cpu: ${TESTER_CORES}
+                        memory: ${TESTER_MEMORY}
+                        ephemeral-storage: 300Gi
+                      limits:
+                        cpu: ${TESTER_CORES}
+                        memory: ${TESTER_MEMORY}
+                        ephemeral-storage: 300Gi
+                    imagePullPolicy: Always
+                    volumeMounts:
+                    - name: dshm
+                      mountPath: /dev/shm
+                    - name: scratch-trt-llm-data
+                      mountPath: /scratch.trt_llm_data
+                      readOnly: true
+                    - name: sw-tensorrt-pvc
+                      mountPath: "/mnt/sw-tensorrt-pvc"
+                      readOnly: false
+                    securityContext:
+                      capabilities:
+                        add:
+                        - SYS_ADMIN"""
+        nodeLabelPrefix = "cpu"
+        break
     default:
         def hasMultipleGPUs = (gpuCount > 1)
         def memorySize = "${TESTER_MEMORY}"
@@ -2925,6 +2958,9 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) {
         // If stageName contains "-AutoDeploy-", add "backend=autodeploy" to makoArgs
         // At this point, only tests with backend=autodeploy or unspecified backend will be run
         makoArgs += ["backend=autodeploy"]
+    } else if (stageName.contains("-Generic-")) {
+        // Generic stages select tests by marker expression rather than backend ownership.
+        makoArgs += ["backend=generic"]
     } else if (stageName.contains("-Verl-")) {
         // If stageName contains "-Verl-", add "backend=verl" to makoArgs
         // At this point, only tests with backend=verl or unspecified backend will be run
@@ -2968,9 +3004,19 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu
     def makoOpts = preDefinedMakoOpts
 
     if (!makoOpts) {
-        def scriptPath = "${llmSrc}/tests/integration/defs/sysinfo/get_sysinfo.py"
         def makoArgs = getMakoArgsFromStageName(stageName)
-        makoOpts = getMakoOpts(scriptPath, makoArgs)
+        if (stageName.startsWith("CPU-")) {
+            def cpuName = env.targetArch == AARCH64_TRIPLE ? "aarch64" : "x86_64"
+            makoOpts = transformMakoArgsToJson(
+                ["Mako options:"] + makoArgs + [
+                    "system_gpu_count=0",
+                    "cpu=${cpuName}",
+                    "linux_distribution_name=ubuntu"
+                ])
+        } else {
+            def scriptPath = "${llmSrc}/tests/integration/defs/sysinfo/get_sysinfo.py"
+            makoOpts = getMakoOpts(scriptPath, makoArgs)
+        }
     }
 
     sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7"
@@ -3565,7 +3611,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         sh "nproc && free -g && hostname"
         echoNodeAndGpuInfo(pipeline, stageName)
         sh "cat ${MODEL_CACHE_DIR}/README"
-        sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m"
+        if (stageName.startsWith("CPU-")) {
+            sh "ln -s /usr/local/cuda/compat/lib.real /usr/local/cuda/compat/lib"
+        } else {
+            sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m"
+        }
         sh "df -h"
 
         // setup HF_HOME to cache model and datasets
@@ -4533,6 +4583,7 @@ def launchTestJobs(pipeline, testFilter)
     // may break the mapping functionality.
 
     x86TestConfigs = [
+        "CPU-Generic-x86-1": ["cpu", "l0_cpu_x86", 1, 1],
         "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4],
         "A10-PyTorch-1": ["a10", "l0_a10", 1, 2],
         "A10-PyTorch-2": ["a10", "l0_a10", 2, 2],
@@ -4731,6 +4782,7 @@ def launchTestJobs(pipeline, testFilter)
 
     // SBSA machines from the Blossom machine pool
     SBSATestConfigs = [
+        "CPU-Generic-arm-1": ["cpu", "l0_cpu_arm", 1, 1],
         "GH200-TensorRT-Post-Merge-1": ["gh200", "l0_gh200", 1, 1],
         // DGX Spark is also named as GB10 Grace Blackwell Superchip.
         "GB10-PyTorch-1": ["gb10x", "l0_gb10", 1, 1],

tensorrt_llm/_torch/attention_backend/flashinfer.py

@@ -29,14 +29,14 @@
                         CustomAttentionMask, MLAParams, PredefinedAttentionMask,
                         merge_attention_forward_args)
 
-try:
-    check_cuda_arch()
-except RuntimeError:
-    # Override TORCH_CUDA_ARCH_LIST for JIT compilation of flashinfer kernels
-    # since the existed TORCH_CUDA_ARCH_LIST may be too general and flashinfer requires sm75+.
-    # Guard on a visible GPU: with CUDA_VISIBLE_DEVICES="" (pure client) the
-    # capability query would force a CUDA context at import time.
-    if torch.cuda.is_available():
+# Guard on a visible GPU: with CUDA_VISIBLE_DEVICES="" (pure client) the
+# check would force a CUDA context at import time.
+if torch.cuda.is_available() and torch.cuda.device_count() > 0:
+    try:
+        check_cuda_arch()
+    except RuntimeError:
+        # Override TORCH_CUDA_ARCH_LIST for JIT compilation of flashinfer kernels
+        # since the existed TORCH_CUDA_ARCH_LIST may be too general and flashinfer requires sm75+.
         capability = torch.cuda.get_device_capability()
         arch_list = f"{capability[0]}.{capability[1]}"
         os.environ["TORCH_CUDA_ARCH_LIST"] = arch_list

tensorrt_llm/_torch/auto_deploy/llm_args.py

@@ -16,7 +16,6 @@
 from pathlib import Path
 from typing import Any, Dict, Literal, Optional, Type, Union
 
-import torch
 from pydantic import Field, ValidationInfo, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -27,6 +26,7 @@
     TorchLlmArgs,
     _ParallelConfig,
 )
+from tensorrt_llm.llmapi.utils import get_device_count
 
 from . import config as _ad_config_pkg
 from .models import ModelFactory, ModelFactoryRegistry
@@ -89,7 +89,7 @@ class LlmArgs(DynamicYamlMixInForSettings, TorchLlmArgs, BaseSettings):
     )
 
     gpus_per_node: int = Field(
-        default=torch.cuda.device_count(),
+        default=get_device_count(),
         description="The number of GPUs per node.",
         frozen=True,
     )

tensorrt_llm/_torch/cuda_tile_utils.py

@@ -44,7 +44,7 @@ def ceil_div(a, b):
     return (a + b - 1) // b
 
 
-if platform.system() != "Windows":
+if platform.system() != "Windows" and torch.cuda.device_count() > 0:
     try:
         import cuda.tile  # noqa: F401
     except ImportError:

tensorrt_llm/_utils.py

@@ -806,10 +806,16 @@ def release_gc():
         torch.cuda.ipc_collect()
 
 
-@lru_cache(maxsize=1)
-def get_sm_version():
-    prop = torch.cuda.get_device_properties(0)
-    return prop.major * 10 + prop.minor
+if torch.cuda.device_count() == 0:
+
+    def get_sm_version():
+        return -1
+else:
+
+    @lru_cache(maxsize=1)
+    def get_sm_version():
+        prop = torch.cuda.get_device_properties(0)
+        return prop.major * 10 + prop.minor
 
 
 @lru_cache(maxsize=1)
@@ -1379,7 +1385,7 @@ def prefer_pinned() -> bool:
     pageable (and not pinned) memory across the board is preferred in CC mode
     to maintain asynchronous execution.
     """
-    return not confidential_compute_enabled()
+    return torch.cuda.device_count() > 0 and not confidential_compute_enabled()
 
 
 def maybe_pin_memory(tensor: torch.Tensor) -> torch.Tensor:

tensorrt_llm/llmapi/utils.py

@@ -149,7 +149,12 @@ def get_device_count() -> int:
     return torch.cuda.device_count() if torch.cuda.is_available() else 0
 
 
-def get_total_gpu_memory(device: int) -> float:
+def get_total_gpu_memory(device: int) -> int:
+    # Compat for no GPU environment, only for device=0.
+    # Otherwise, the caller should ensure there are that many GPUs.
+    if device == 0 and get_device_count() == 0:
+        return 0
+
     return torch.cuda.get_device_properties(device).total_memory
 
 

tensorrt_llm/profiler.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import contextlib
 import time
 from functools import partial
 from typing import Literal, Optional, Tuple, Union
@@ -110,18 +111,24 @@ def summary():
 MemUnitType = Literal["GiB", "MiB", "KiB"]
 
 
-class PyNVMLContext:
-    def __enter__(self):
-        if pynvml is not None:
+@contextlib.contextmanager
+def pynvml_context():
+    has_pynvml = pynvml is not None
+    if has_pynvml:
+        try:
             pynvml.nvmlInit()
+        except pynvml.NVMLError:
+            has_pynvml = False
 
-    def __exit__(self, type, value, traceback):
-        if pynvml is not None:
+    try:
+        yield
+    finally:
+        if has_pynvml:
             pynvml.nvmlShutdown()
 
 
 if pynvml is not None:
-    with PyNVMLContext():
+    with pynvml_context():
         _device_get_memory_info_fn = partial(
             pynvml.nvmlDeviceGetMemoryInfo,
             version=pynvml.nvmlMemory_v2,
@@ -147,7 +154,7 @@ def device_memory_info(device: Optional[Union[torch.device, int]] = None) -> Tup
         if device is None:
             device = torch.cuda.current_device()
         index = device.index if isinstance(device, torch.device) else device
-        with PyNVMLContext():
+        with pynvml_context():
             handle = pynvml.nvmlDeviceGetHandleByIndex(index)
             mem_info = _device_get_memory_info_fn(handle)
         return mem_info.used, mem_info.free, mem_info.total

tests/integration/defs/conftest.py

@@ -1807,13 +1807,17 @@ def get_gpu_device_list():
         suffix = ".exe" if is_windows() else ""
         # TODO: Use NRSU because we can't assume nvidia-smi across all platforms.
         cmd = " ".join(["nvidia-smi" + suffix, "-L"])
-        output = check_output(cmd, shell=True, cwd=temp_dirname)
+        try:
+            output = check_output(cmd, shell=True, cwd=temp_dirname)
+        except sp.CalledProcessError:
+            return []
     return [l.strip() for l in output.strip().split("\n")]
 
 
 def check_device_contain(keyword_list):
     "check device not contain keyword"
-    device = get_gpu_device_list()[0]
+    devices = get_gpu_device_list()
+    device = devices[0] if devices else ""
     return any(keyword in device for keyword in keyword_list)
 
 
@@ -2008,6 +2012,12 @@ def pytest_addoption(parser):
         help=
         "Enable Ray orchestrator path for integration tests (disables MPI).",
     )
+    parser.addoption(
+        "--unittest-markexpr",
+        action="store",
+        default=None,
+        help="Marker expression forwarded to nested unittest pytest runs.",
+    )
     parser.addoption(
         "--perf-log-formats",
         help=

tests/integration/defs/test_unittests.py

@@ -66,7 +66,12 @@ def merge_attr(name, type_=int):
 def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request):
     import pandas as pd
     import pynvml
-    pynvml.nvmlInit()
+    try:
+        pynvml.nvmlInit()
+        gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+        gpu_name = pynvml.nvmlDeviceGetName(gpu_handle)
+    except pynvml.NVMLError:
+        gpu_name = "CPU"
 
     test_root = tests_path()
     dry_run = False
@@ -79,6 +84,7 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request):
 
     waives_file = request.config.getoption("--waives-file")
     run_ray = request.config.getoption("--run-ray")
+    unittest_markexpr = request.config.getoption("--unittest-markexpr")
 
     num_workers = 1
 
@@ -87,8 +93,6 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request):
     agg_unit_mem_path = f'{test_root}/integration/defs/agg_unit_mem_df.csv'
     print(f'Loading unittest parallel config from: {agg_unit_mem_path}')
     agg_unit_mem_df = pd.read_csv(agg_unit_mem_path)
-    gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0)
-    gpu_name = pynvml.nvmlDeviceGetName(gpu_handle)
     if isinstance(gpu_name, bytes):
         gpu_name = gpu_name.decode()
     print(f'GPU name from NVML (index 0): {gpu_name!r}')
@@ -167,6 +171,9 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request):
     if run_ray:
         command += ["--run-ray"]
 
+    if unittest_markexpr:
+        command += ["-m", unittest_markexpr]
+
     s3_secret_key = None
     s3_upload_path = request.config.getoption("--s3-upload-path", default=None)
     if s3_upload_path:

tests/integration/test_lists/test-db/l0_a10.yml

@@ -139,7 +139,6 @@ l0_a10:
   - unittest/llmapi/test_additional_model_outputs.py -m "gpu1"
   - unittest/llmapi/test_request_priority.py
   # executor
-  - unittest/executor/test_rpc.py
   - unittest/executor/test_ipc.py
   - unittest/executor/test_fatal_error_health_check.py
   - unittest/executor/test_postprocessor_hook.py

tests/integration/test_lists/test-db/l0_cpu_arm.yml

@@ -0,0 +1,16 @@
+version: 0.0.1
+l0_cpu_arm:
+- condition:
+    ranges:
+      system_gpu_count:
+        gte: 0
+        lte: 0
+    wildcards:
+      linux_distribution_name: ubuntu*
+      cpu: aarch64
+    terms:
+      stage: pre_merge
+      backend: generic
+      orchestrator: mpi
+  tests:
+  - unittest/executor/test_rpc.py

tests/integration/test_lists/test-db/l0_cpu_x86.yml

@@ -0,0 +1,16 @@
+version: 0.0.1
+l0_cpu_x86:
+- condition:
+    ranges:
+      system_gpu_count:
+        gte: 0
+        lte: 0
+    wildcards:
+      linux_distribution_name: ubuntu*
+      cpu: x86_64
+    terms:
+      stage: pre_merge
+      backend: generic
+      orchestrator: mpi
+  tests:
+  - unittest/executor/test_rpc.py

tests/unittest/auto_deploy/_utils_test/_torch_test_utils.py

@@ -41,10 +41,14 @@ def reset_parameters(model: torch.nn.Module):
 
 
 def fp8_compatible():
+    if torch.cuda.device_count() == 0:
+        return False
     return torch.cuda.get_device_capability(0) >= (8, 9)
 
 
 def fp4_compatible():
+    if torch.cuda.device_count() == 0:
+        return False
     return torch.cuda.get_device_capability(0) >= (10, 0)