diff --git a/README.md b/README.md index 1f9e7fa262b7..6f26d9665339 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ TensorRT LLM [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/NVIDIA/TensorRT-LLM) [![python](https://img.shields.io/badge/python-3.12-green)](https://www.python.org/downloads/release/python-3123/) [![python](https://img.shields.io/badge/python-3.10-green)](https://www.python.org/downloads/release/python-31012/) -[![cuda](https://img.shields.io/badge/cuda-13.1.1-green)](https://developer.nvidia.com/cuda-downloads) -[![torch](https://img.shields.io/badge/torch-2.10.0-green)](https://pytorch.org) +[![cuda](https://img.shields.io/badge/cuda-13.2.1-green)](https://developer.nvidia.com/cuda-downloads) +[![torch](https://img.shields.io/badge/torch-2.11.0-green)](https://pytorch.org) [![version](https://img.shields.io/badge/release-1.3.0rc18-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) [![license](https://img.shields.io/badge/license-Apache%202-blue)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) diff --git a/constraints.txt b/constraints.txt index 519a9a29f9fc..09a933566229 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,11 +1,5 @@ # These vulnerabilities were inherited from the base image (pytorch:25.12-py3) and should be removed when the base image # is updated. -# WAR against https://github.com/advisories/GHSA-8rrh-rw8j-w5fx -wheel>=0.46.2 -# WAR against https://github.com/advisories/GHSA-qjxf-f2mg-c6mc -tornado>=6.5.5 -# WAR against https://github.com/advisories/GHSA-3936-cmfr-pm3m -black>=26.3.1 # Upgrade base image nvidia-cutlass-dsl 4.3.5 to 4.4.2 nvidia-cutlass-dsl>=4.4.2 # The `nvidia-cutlass-dsl` package does not pin numpy at all, which can be problematic in certain CI diff --git a/cpp/include/tensorrt_llm/runtime/virtualMemory.h b/cpp/include/tensorrt_llm/runtime/virtualMemory.h index a7e95b42d707..d74673f37e01 100644 --- a/cpp/include/tensorrt_llm/runtime/virtualMemory.h +++ b/cpp/include/tensorrt_llm/runtime/virtualMemory.h @@ -505,10 +505,7 @@ class CudaVirtualMemoryAllocator { std::size_t gpuAlignment = 1; CUmemAllocationProp const prop{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, - { - CU_MEM_LOCATION_TYPE_DEVICE, - device, - }}; + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {device}}}; TLLM_CU_CHECK( cuMemGetAllocationGranularity(&gpuAlignment, &prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); alignment = std::lcm(getpagesize(), gpuAlignment); diff --git a/cpp/tensorrt_llm/deep_ep/CMakeLists.txt b/cpp/tensorrt_llm/deep_ep/CMakeLists.txt index 562c9e7d694c..e00815eed13e 100644 --- a/cpp/tensorrt_llm/deep_ep/CMakeLists.txt +++ b/cpp/tensorrt_llm/deep_ep/CMakeLists.txt @@ -120,6 +120,15 @@ if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_C_COMPILER gcc) set(CMAKE_CXX_COMPILER g++) set(CMAKE_CUDA_HOST_COMPILER g++) + # PyTorch's cmake/public/cuda.cmake (loaded transitively by + # find_package(Torch)) appends -Xcompiler=-fclang-abi-compat=17 to + # CMAKE_CUDA_FLAGS whenever the parent build is configured with Clang>=18 (see + # pytorch PR #175233). Since this subdirectory falls back to GCC for NVSHMEM + # compatibility, that Clang-only flag would be forwarded to g++ via `nvcc + # -ccbin=g++` and abort the build with: g++: error: unrecognized command-line + # option '-fclang-abi-compat=17' + string(REGEX REPLACE "-Xcompiler=-fclang-abi-compat=[0-9]+" "" + CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") endif() # Add nvshmem external project @@ -204,6 +213,12 @@ target_compile_options( target_compile_definitions( deep_ep_cpp_tllm PRIVATE DISABLE_AGGRESSIVE_PTX_INSTRS TORCH_EXTENSION_NAME=deep_ep_cpp_tllm) +# Newer CUDA containers provide NVSHMEM headers in the default CUDA include +# directory. DeepEP must compile against the vendored NVSHMEM headers because it +# links the vendored NVSHMEM static library below. +target_include_directories( + deep_ep_cpp_tllm BEFORE + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/nvshmem-build/src/include) target_link_libraries( deep_ep_cpp_tllm PRIVATE nvshmem_project::nvshmem ${TORCH_LIBRARIES} ${TORCH_PYTHON_LIB}) diff --git a/cpp/tensorrt_llm/flash_mla/CMakeLists.txt b/cpp/tensorrt_llm/flash_mla/CMakeLists.txt index e87f12275f60..024cee521d10 100644 --- a/cpp/tensorrt_llm/flash_mla/CMakeLists.txt +++ b/cpp/tensorrt_llm/flash_mla/CMakeLists.txt @@ -44,6 +44,15 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64" AND CMAKE_CXX_COMPILER_ID set(CMAKE_CUDA_HOST_COMPILER ${GCC_EXECUTABLE}) message( STATUS "FlashMLA: Using GCC at ${GCC_EXECUTABLE} for CUDA compilation") + # PyTorch's cmake/public/cuda.cmake (loaded transitively by + # find_package(Torch)) appends -Xcompiler=-fclang-abi-compat=17 to + # CMAKE_CUDA_FLAGS whenever the parent build is configured with Clang>=18 (see + # pytorch PR #175233). Since CUDA host compilation here falls back to GCC, + # that Clang-only flag would be forwarded to g++ via `nvcc -ccbin=g++` and + # abort the build with: g++: error: unrecognized command-line option + # '-fclang-abi-compat=17' + string(REGEX REPLACE "-Xcompiler=-fclang-abi-compat=[0-9]+" "" + CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") endif() # Check CUDA version and architecture support diff --git a/cpp/tensorrt_llm/runtime/virtualMemory.cpp b/cpp/tensorrt_llm/runtime/virtualMemory.cpp index 9b23866d6281..0d08012a29d8 100644 --- a/cpp/tensorrt_llm/runtime/virtualMemory.cpp +++ b/cpp/tensorrt_llm/runtime/virtualMemory.cpp @@ -344,11 +344,7 @@ void CudaVirtualMemoryAllocator::allocate(Pointer* ptr, std::size_t n, int devic CUDAVirtualMemoryChunk::Configurators configurators; configurators.push_back(std::make_unique(address, alignedSize, - CUmemAccessDesc{{ - CU_MEM_LOCATION_TYPE_DEVICE, - device, - }, - CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); + CUmemAccessDesc{CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {device}}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); switch (mConfig->mMode) { @@ -368,10 +364,7 @@ void CudaVirtualMemoryAllocator::allocate(Pointer* ptr, std::size_t n, int devic mConfig->mManager.add(address, mConfig->mTag, std::make_unique>(CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, - { - CU_MEM_LOCATION_TYPE_DEVICE, - device, - }}, + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {device}}}, alignedSize), std::move(configurators)); diff --git a/cpp/tests/unit_tests/common/cudaDriverWrapperTest.cpp b/cpp/tests/unit_tests/common/cudaDriverWrapperTest.cpp index 2ba48cd9fb81..0c99fb955cd7 100644 --- a/cpp/tests/unit_tests/common/cudaDriverWrapperTest.cpp +++ b/cpp/tests/unit_tests/common/cudaDriverWrapperTest.cpp @@ -31,7 +31,7 @@ TEST(TestCudaDriverWrapper, TllmCuCheckFailingWithValidParametersDoesNotThrow) CUmemAllocationHandleType::CU_MEM_HANDLE_TYPE_NONE, CUmemLocation{ CUmemLocationType::CU_MEM_LOCATION_TYPE_DEVICE, - 0, + {0}, }, nullptr}; auto const granularity = tensorrt_llm::common::getAllocationGranularity(); @@ -51,7 +51,7 @@ TEST(TestCudaDriverWrapper, TllmCuCheckFailingWithInvalidParametersThrows) CUmemAllocationHandleType::CU_MEM_HANDLE_TYPE_NONE, CUmemLocation{ CUmemLocationType::CU_MEM_LOCATION_TYPE_DEVICE, - 0, + {0}, }, nullptr}; ASSERT_THROW(TLLM_CU_CHECK(cuMemCreate(&handle, -1, &prop, 0ULL)), tensorrt_llm::common::TllmException); diff --git a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp index 85d49433d882..f2045e7659d1 100644 --- a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp +++ b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp @@ -142,19 +142,12 @@ TEST_F(VirtualMemoryTest, TestBasic) CUDAVirtualMemoryChunk::CreatorPtr creator = std::make_unique>(CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, - { - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }}, + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}}, size); CUDAVirtualMemoryChunk::Configurators configurators; configurators.push_back(std::make_unique(address, size, - CUmemAccessDesc{{ - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }, - CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); + CUmemAccessDesc{CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); CUDAVirtualMemoryChunk vm(std::move(creator), std::move(configurators)); ASSERT_EQ(vm.status(), CUDAVirtualMemoryChunk::RELEASED); @@ -212,19 +205,12 @@ TEST_P(VirtualMemoryOffloadConfigurator, Test) CUDAVirtualMemoryChunk::CreatorPtr creator = std::make_unique>(CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, - { - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }}, + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}}, size); CUDAVirtualMemoryChunk::Configurators configurators; configurators.push_back(std::make_unique(address, size, - CUmemAccessDesc{{ - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }, - CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); + CUmemAccessDesc{CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); configurators.push_back(std::make_unique(address, size, backType, stream.get(), false)); CUDAVirtualMemoryChunk vm(std::move(creator), std::move(configurators)); @@ -611,14 +597,14 @@ TEST_F(VirtualMemoryTest, TestFacilities) { // Create original CUDAVirtualMemoryChunk - CUDAVirtualMemoryChunk::CreatorPtr creator - = std::make_unique>(CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, - CU_MEM_HANDLE_TYPE_NONE, {CU_MEM_LOCATION_TYPE_DEVICE, 0}}, - size); + CUDAVirtualMemoryChunk::CreatorPtr creator = std::make_unique>( + CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}}, + size); CUDAVirtualMemoryChunk::Configurators configurators; - configurators.push_back(std::make_unique( - address, size, CUmemAccessDesc{{CU_MEM_LOCATION_TYPE_DEVICE, 0}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); + configurators.push_back(std::make_unique(address, size, + CUmemAccessDesc{CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); CUDAVirtualMemoryChunk original(std::move(creator), std::move(configurators)); original.materialize(); @@ -977,19 +963,12 @@ TEST_F(VirtualMemoryManagerTest, TestBasic) CUDAVirtualMemoryChunk::CreatorPtr creator = std::make_unique>(CUmemAllocationProp{CU_MEM_ALLOCATION_TYPE_PINNED, CU_MEM_HANDLE_TYPE_NONE, - { - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }}, + CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}}, size); CUDAVirtualMemoryChunk::Configurators configurators; configurators.push_back(std::make_unique(address, size, - CUmemAccessDesc{{ - CU_MEM_LOCATION_TYPE_DEVICE, - 0, - }, - CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); + CUmemAccessDesc{CUmemLocation{CU_MEM_LOCATION_TYPE_DEVICE, {0}}, CU_MEM_ACCESS_FLAGS_PROT_READWRITE})); auto memoryBegin = getCurrentProcessMemoryInfo(); diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index d4bf170470ac..34a7db7b8b5d 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -1,8 +1,8 @@ # Multi-stage Dockerfile ARG BASE_IMAGE=nvcr.io/nvidia/pytorch ARG TRITON_IMAGE=nvcr.io/nvidia/tritonserver -ARG BASE_TAG=26.02-py3 -ARG TRITON_BASE_TAG=26.02-py3 +ARG BASE_TAG=26.04-py3 +ARG TRITON_BASE_TAG=26.04-py3 ARG DEVEL_IMAGE=devel FROM ${BASE_IMAGE}:${BASE_TAG} AS base diff --git a/docker/Makefile b/docker/Makefile index 9a9c44ce8b56..0ca78eef80a0 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -202,21 +202,21 @@ jenkins-rockylinux8_%: PYTHON_VERSION_TAG_ID = $(if $(findstring 3.12,${PYTHON_V jenkins-rockylinux8_%: IMAGE_WITH_TAG = $(shell . ../jenkins/current_image_tags.properties && echo $$LLM_ROCKYLINUX8_${PYTHON_VERSION_TAG_ID}_DOCKER_IMAGE) jenkins-rockylinux8_%: STAGE = tritondevel jenkins-rockylinux8_%: BASE_IMAGE = nvcr.io/nvidia/cuda -jenkins-rockylinux8_%: BASE_TAG = 13.1.1-devel-rockylinux8 +jenkins-rockylinux8_%: BASE_TAG = 13.2.1-devel-rockylinux8 rockylinux8_%: STAGE = tritondevel rockylinux8_%: BASE_IMAGE = nvcr.io/nvidia/cuda -rockylinux8_%: BASE_TAG = 13.1.1-devel-rockylinux8 +rockylinux8_%: BASE_TAG = 13.2.1-devel-rockylinux8 # For x86_64 ubuntu22_%: STAGE = tritondevel ubuntu22_%: BASE_IMAGE = nvcr.io/nvidia/cuda -ubuntu22_%: BASE_TAG = 13.1.1-devel-ubuntu22.04 +ubuntu22_%: BASE_TAG = 13.2.1-devel-ubuntu22.04 # For x86_64 and aarch64 ubuntu24_%: STAGE = tritondevel ubuntu24_%: BASE_IMAGE = nvcr.io/nvidia/cuda -ubuntu24_%: BASE_TAG = 13.1.1-devel-ubuntu24.04 +ubuntu24_%: BASE_TAG = 13.2.1-devel-ubuntu24.04 trtllm_%: STAGE = release trtllm_%: PUSH_TO_STAGING := 0 diff --git a/docker/common/install_cuda_toolkit.sh b/docker/common/install_cuda_toolkit.sh index 512a1043fb8e..41494e617c04 100644 --- a/docker/common/install_cuda_toolkit.sh +++ b/docker/common/install_cuda_toolkit.sh @@ -5,7 +5,7 @@ set -ex # This script is used for reinstalling CUDA on Rocky Linux 8 with the run file. # CUDA version is usually aligned with the latest NGC CUDA image tag. # Only use when public CUDA image is not ready. -CUDA_VER="13.1.1_590.48.01" +CUDA_VER="13.2.1_595.58.03" CUDA_VER_SHORT="${CUDA_VER%_*}" NVCC_VERSION_OUTPUT=$(nvcc --version) diff --git a/docker/common/install_pytorch.sh b/docker/common/install_pytorch.sh index 5419bcc6de9b..4ec997fdbeca 100644 --- a/docker/common/install_pytorch.sh +++ b/docker/common/install_pytorch.sh @@ -4,8 +4,8 @@ set -ex # Use latest stable version from https://pypi.org/project/torch/#history # and closest to the version specified in -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-02.html#rel-26-02 -TORCH_VERSION="2.10.0" +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 +TORCH_VERSION="2.11.0" SYSTEM_ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') prepare_environment() { diff --git a/docker/common/install_tensorrt.sh b/docker/common/install_tensorrt.sh index 9ae8be04cb21..66bf24deb2b4 100644 --- a/docker/common/install_tensorrt.sh +++ b/docker/common/install_tensorrt.sh @@ -2,20 +2,20 @@ set -ex -TRT_VER="10.15.1.29" +TRT_VER="10.16.1.11" # Align with the pre-installed cuDNN / cuBLAS / NCCL versions from -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-02.html#rel-26-02 -CUDA_VER="13.1" # 13.1.1 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 +CUDA_VER="13.2" # 13.2.1 # Keep the installation for cuDNN if users want to install PyTorch with source codes. # PyTorch 2.x can compile with cuDNN v9. -CUDNN_VER="9.19.0.56-1" -NCCL_VER="2.29.2-1+cuda13.1" -CUBLAS_VER="13.2.1.1-1" +CUDNN_VER="9.21.0.82-1" +NCCL_VER="2.29.7-1+cuda13.2" +CUBLAS_VER="13.4.0.1-1" # Align with the pre-installed CUDA / NVCC / NVRTC versions from # https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html -NVRTC_VER="13.1.115-1" -CUDA_RUNTIME="13.1.80-1" -CUDA_DRIVER_VERSION="590.48.01-1.el8" +NVRTC_VER="13.2.78-1" +CUDA_RUNTIME="13.2.75-1" +CUDA_DRIVER_VERSION="595.58.03-1.el8" for i in "$@"; do case $i in diff --git a/docs/source/legacy/reference/support-matrix.md b/docs/source/legacy/reference/support-matrix.md index 4c7629843299..e63f439b9446 100644 --- a/docs/source/legacy/reference/support-matrix.md +++ b/docs/source/legacy/reference/support-matrix.md @@ -158,9 +158,9 @@ The following table shows the supported software for TensorRT-LLM. * - - Software Compatibility * - Container - - [26.02](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html) + - [26.04](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html) * - TensorRT - - [10.14](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html) + - [10.16](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html) * - Precision - - Blackwell (SM100/SM103/SM120) - FP32, FP16, BF16, FP8, FP4, INT8, INT4 diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 405280496af8..66c62c1b1580 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -408,7 +408,7 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) def llmPath = sh (script: "realpath ${LLM_ROOT}",returnStdout: true).trim() // TODO: Remove after the cmake version is upgraded to 3.31.8 // Get triton tag from docker/dockerfile.multi - def tritonShortTag = "r26.02" + def tritonShortTag = "r26.04" sh "cd ${LLM_ROOT}/triton_backend/inflight_batcher_llm && mkdir build && cd build && cmake .. -DTRTLLM_DIR=${llmPath} -DTRITON_COMMON_REPO_TAG=${tritonShortTag} -DTRITON_CORE_REPO_TAG=${tritonShortTag} -DTRITON_THIRD_PARTY_REPO_TAG=${tritonShortTag} -DTRITON_BACKEND_REPO_TAG=${tritonShortTag} -DUSE_CXX11_ABI=ON && make -j${buildJobs} install" // Step 3: packaging wheels into tarfile diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fbdbd4df3091..0bfb7418a8a1 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1,4 +1,4 @@ -@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ +@Library(['bloom-jenkins-shared-lib@emma/update_nsc_login_node', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException import groovy.transform.Field @@ -46,7 +46,7 @@ LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = env.wheelDockerImagePy312 LLM_WHEEL_DOCKER_IMAGE = env.wheelDockerImage // DLFW torch image -DLFW_IMAGE = "urm.nvidia.com/docker/nvidia/pytorch:26.02-py3" +DLFW_IMAGE = "urm.nvidia.com/docker/nvidia/pytorch:26.04-py3" //Ubuntu base image UBUNTU_22_04_IMAGE = "urm.nvidia.com/docker/ubuntu:22.04" @@ -1293,7 +1293,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG """ } else { if(nodeCount > 1) { - srunArgs.add("--mpi=pmi2") + srunArgs.add("--mpi=pmix") } def scriptContent = """ @@ -2293,7 +2293,7 @@ def launchTestListCheck(pipeline) def llmPath = sh (script: "realpath .", returnStdout: true).trim() def llmSrc = "${llmPath}/TensorRT-LLM/src" trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install -r ${llmSrc}/requirements-dev.txt") - sh "NVIDIA_TRITON_SERVER_VERSION=26.02 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive" + sh "NVIDIA_TRITON_SERVER_VERSION=26.04 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive" } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -4119,7 +4119,9 @@ def launchTestJobs(pipeline, testFilter) "DGX_B200-8_GPUs-PyTorch-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 3, 8, 1, true], "DGX_B200-8_GPUs-PyTorch-3": ["auto:dgx-b200-flex", "l0_dgx_b200", 3, 3, 8, 1, true], "DGX_B200-8_GPUs-AutoDeploy-Post-Merge-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 1, 8, 1, true], - "DGX_B200-4_GPUs-Verl-Post-Merge-1": ["auto:dgx-b200-flex", "l0_verl", 1, 1, 4, 1, true], + // Disable Verl stage due to https://nvbugs/6236818. + // Please re-enable it after the bug is fixed. + // "DGX_B200-4_GPUs-Verl-Post-Merge-1": ["auto:dgx-b200-flex", "l0_verl", 1, 1, 4, 1, true], "B300-PyTorch-1": ["auto:dgx-b300-flex", "l0_b300", 1, 2, 1, 1, true], "B300-PyTorch-2": ["auto:dgx-b300-flex", "l0_b300", 2, 2, 1, 1, true], "DGX_B300-4_GPUs-PyTorch-1": ["auto:dgx-b300-flex", "l0_dgx_b300", 1, 1, 4, 1, true], @@ -4590,14 +4592,14 @@ def launchTestJobs(pipeline, testFilter) def platform = cpu_arch == X86_64_TRIPLE ? "x86_64" : "sbsa" trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-1") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-2") } - // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) + // Extra PyTorch CUDA 13.2 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { - echo "###### Extra PyTorch CUDA 13.0 install Start ######" + echo "###### Extra PyTorch CUDA 13.2 install Start ######" // Use internal mirror instead of https://download.pytorch.org/whl/cu130 for better network stability. // PyTorch CUDA 13.0 package and torchvision package can be installed as expected. - trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install torch==2.10.0+cu130 torchvision==0.25.0+cu130 --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/pytorch-cu128-remote/simple --extra-index-url https://download.pytorch.org/whl/cu130") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install torch==2.11.0+cu130 torchvision==0.26.0+cu130 --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/pytorch-cu128-remote/simple --extra-index-url https://download.pytorch.org/whl/cu130") } def libEnv = [] diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index e220c19ae3bd..b2a4cd876b2b 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-x86_64-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202606051544-14972 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-sbsa-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202606051544-14972 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py310-trt10.15.1.29-skip-tritondevel-202606051544-14972 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py312-trt10.15.1.29-skip-tritondevel-202606051544-14972 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.15.1.29-skip-tritondevel-202606051544-14972 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606091350-12643 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-aarch64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606091350-12643 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202606091350-12643 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202606091350-12643 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202606091350-12643 diff --git a/jenkins/scripts/perf/README.md b/jenkins/scripts/perf/README.md index 5e68643ec06d..8c2f89679615 100644 --- a/jenkins/scripts/perf/README.md +++ b/jenkins/scripts/perf/README.md @@ -270,9 +270,16 @@ and benchmark srun steps never see MPI flags. **Where MPI is configured:** - `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` — `--mpi=pmix` on ctx/gen srun commands only -- `jenkins/scripts/perf/local/submit.py` — `--mpi=pmi2` for aggregated mode only, +- `jenkins/scripts/perf/local/submit.py` — `--mpi=pmix` for aggregated mode only, no MPI flag for disaggregated mode (handled by the draft template) -- `jenkins/L0_Test.groovy` — `--mpi=pmi2` for non-disagg multi-node only +- `jenkins/L0_Test.groovy` — `--mpi=pmix` for non-disagg multi-node only + +> The DLFW base image (PyTorch 26.03+) ships an Open MPI build with PMIx support +> only and **no classic PMI-2**. Under `--mpi=pmi2` srun exposes a PMI-2 server +> but no PMIx server, so OMPI's `pmix3x_client` fails with +> `OPAL ERROR: Unreachable in pmix3x_client.c at line 111` and aborts in +> `MPI_Init_thread`. `--mpi=pmix` matches what the disagg path has always used +> and works against the new OMPI build. ### Key Rules diff --git a/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh b/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh index 0421439d8236..e8daf893f87b 100644 --- a/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh +++ b/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh @@ -12,7 +12,7 @@ chmod +x $runScript # Run aggregated test echo "Starting aggregated test..." world_size=${world_size:-$((totalNodes * gpusPerNodePerServer))} -if ! srun "${srunArgs[@]}" --mpi=pmi2 --kill-on-bad-exit=1 \ +if ! srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \ -N $totalNodes \ --ntasks=$world_size \ --ntasks-per-node=$gpusPerNodePerServer \ diff --git a/jenkins/scripts/perf/local/slurm_run.sh b/jenkins/scripts/perf/local/slurm_run.sh index d10dc6fc23a8..a874b4fb965d 100755 --- a/jenkins/scripts/perf/local/slurm_run.sh +++ b/jenkins/scripts/perf/local/slurm_run.sh @@ -15,6 +15,15 @@ fi cd $llmSrcNode/tests/integration/defs +# Force PMIx to use the in-memory hash GDS instead of ds12/ds21 shared-memory. +# Under `srun --mpi=pmix` with the DLFW 26.04 OpenMPI build, the shared-memory +# GDS modes can fail to publish UCX worker addresses across nodes, producing: +# pml_ucx.c:178 Error: Failed to receive UCX worker address: Not found (-13) +# pml_ucx.c:482 Error: Failed to resolve UCX endpoint for rank N +# See https://github.com/open-mpi/ompi/issues/6981. Setting this is a no-op +# when PMIx isn't used. +export PMIX_MCA_gds=hash + # Turn off "exit on error" so the following lines always run set +e diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py index 97527eb9a8c4..fbb57182e810 100755 --- a/jenkins/scripts/perf/local/submit.py +++ b/jenkins/scripts/perf/local/submit.py @@ -368,7 +368,7 @@ def generate_srun_args(args, runtime_mode, timestamp, llm_src="", hardware_confi lines.append("--container-env=NVIDIA_IMEX_CHANNELS") - # Single-GPU aggregated jobs run one process without MPI -- drop --mpi=pmi2 + # Single-GPU aggregated jobs run one process without MPI -- drop --mpi=pmix is_single_gpu_aggr = ( is_aggr and hardware_config is not None and hardware_config.get("total_gpus") == 1 ) @@ -376,7 +376,7 @@ def generate_srun_args(args, runtime_mode, timestamp, llm_src="", hardware_confi if args.mpi_type: lines.append(f"--mpi={args.mpi_type}") elif is_aggr and not is_single_gpu_aggr: - lines.append("--mpi=pmi2") + lines.append("--mpi=pmix") return lines @@ -561,7 +561,7 @@ def main(): "--mpi-type", default="", help="MPI type for srun (e.g. pmix, pmi2). If not set, aggregated runs default to" - " --mpi=pmi2; non-aggregated runs omit --mpi entirely.", + " --mpi=pmix; non-aggregated runs omit --mpi entirely.", ) parser.add_argument( "--disagg-server-port", diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index eea9733d2d2d..87b932285229 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -88,6 +88,15 @@ if [ "${UCX_TLS:-}" = "tcp" ]; then unset UCX_TLS echo "Unset UCX_TLS (cluster injected UCX_TLS=tcp)" fi + +# Force PMIx to use the in-memory hash GDS instead of ds12/ds21 shared-memory. +# Under `srun --mpi=pmix` with the DLFW 26.04 OpenMPI build, the shared-memory +# GDS modes can fail to publish UCX worker addresses across nodes, producing: +# pml_ucx.c:178 Error: Failed to receive UCX worker address: Not found (-13) +# pml_ucx.c:482 Error: Failed to resolve UCX endpoint for rank N +# See https://github.com/open-mpi/ompi/issues/6981. Setting this is a no-op +# when PMIx isn't used. +export PMIX_MCA_gds=hash echo "Library Path:" echo "$LD_LIBRARY_PATH" env | sort diff --git a/requirements.txt b/requirements.txt index 9fe0fbf59814..e8d7cab33435 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,14 +21,14 @@ pandas h5py==3.12.1 StrEnum sentencepiece>=0.1.99 -tensorrt~=10.15.1 -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-02.html#rel-26-02 uses 2.11.0a0. -torch>=2.10.0,<=2.11.0a0 +tensorrt~=10.16.1 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 uses 2.12.0a0. +torch>=2.11.0,<=2.12.0a0 torchvision nvidia-modelopt[torch]~=0.37.0 -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-02.html#rel-26-02 uses 2.29.2 -# torch 2.10.0+cu130 depends on nvidia-nccl-cu13==2.28.9 -nvidia-nccl-cu13>=2.28.9,<=2.29.2 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 uses 2.29.7 +# torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 +nvidia-nccl-cu13>=2.28.9,<=2.29.7 nvidia-cuda-nvrtc transformers==5.5.4 prometheus_client @@ -89,3 +89,5 @@ etcd-sdk-python==0.0.7 python-multipart smg-grpc-proto>=0.4.2 cache-dit>=1.3.5 +librosa +msgpack diff --git a/scripts/rename_docker_images.py b/scripts/rename_docker_images.py old mode 100755 new mode 100644 index dacbcb350ea3..a38983398ec4 --- a/scripts/rename_docker_images.py +++ b/scripts/rename_docker_images.py @@ -23,10 +23,6 @@ def parse_arguments() -> _ap.Namespace: parser = _ap.ArgumentParser( description="Rename Docker images based on the given instructions.") - parser.add_argument( - 'src_branch', - type=str, - help="The name of the source branch releasing the Docker image.") parser.add_argument( 'src_build_id', type=int, @@ -52,6 +48,22 @@ def parse_arguments() -> _ap.Namespace: help= "The new stage part of the destination image name (default: tritondevel)." ) + source_group = parser.add_mutually_exclusive_group() + source_group.add_argument( + "--commit", + type=str, + default=None, + help= + "GitHub mode: short commit SHA embedded in the source image tag (default: `git rev-parse --short HEAD`)." + ) + source_group.add_argument( + "--src-branch", + dest="src_branch", + type=str, + default=None, + help="GitLab mode: source branch name embedded in the source image tag. " + "When set, the tag is built as `--` instead of the GitHub-PR layout." + ) return parser.parse_args() @@ -60,6 +72,14 @@ def get_current_timestamp() -> str: return _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d%H%M") +def get_current_commit() -> str: + """Return the 7-char short SHA of the current git HEAD.""" + return _sp.run(["git", "rev-parse", "--short=7", "HEAD"], + check=True, + text=True, + capture_output=True).stdout.strip()[:7] + + def run_shell_command(command: str, dry_run: bool) -> None: """Run a shell command and display its output. @@ -178,29 +198,37 @@ def find_and_replace_in_files(directory, file_extension: str, def rename_images(*, - src_branch: str, src_build_id: int, dst_mr: int, stage: str, + commit: str | None = None, + src_branch: str | None = None, timestamp: str | None = None, dry_run: bool = False) -> None: - print( - f"Renaming images for branch {src_branch} and build id {src_build_id} to {dst_mr}" - ) + if src_branch is not None: + src_branch_sanitized = src_branch.replace("/", "_") + src_suffix = f"{src_branch_sanitized}-{src_build_id}" + print( + f"Renaming images (GitLab mode) from branch {src_branch} build id {src_build_id} to MR {dst_mr}" + ) + else: + commit = (commit or get_current_commit())[:7] + src_suffix = f"{commit}-github-pr-{dst_mr}-{src_build_id}" + print( + f"Renaming images (GitHub mode) at commit {commit} build id {src_build_id} to MR {dst_mr}" + ) if dry_run: print("Dry-run mode enabled. No actual changes will be made.") else: print("Renaming images...") timestamp = timestamp or get_current_timestamp() - src_branch_sanitized = src_branch.replace("/", "_") base_dir = find_script_directory().parent current_tags_path = base_dir / "jenkins" / CURRENT_TAG_FILE for dst_key, src_pattern in IMAGE_MAPPING.items(): print(f"Processing {dst_key} ...") - src_image = f"{src_pattern}-{src_branch_sanitized}-{src_build_id}".replace( - "__stage__", stage) + src_image = f"{src_pattern}-{src_suffix}".replace("__stage__", stage) dst_image_old = extract_line_after_prefix(current_tags_path, dst_key + "=").strip('"') dst_image = replace_text_between_dashes( diff --git a/tensorrt_llm/_torch/compilation/backend.py b/tensorrt_llm/_torch/compilation/backend.py index b952236e3916..43319ac9425c 100644 --- a/tensorrt_llm/_torch/compilation/backend.py +++ b/tensorrt_llm/_torch/compilation/backend.py @@ -5,7 +5,7 @@ import torch import torch._inductor.config as inductor_config from torch._functorch.aot_autograd import aot_module_simplified -from torch._inductor.compile_fx import compile_fx, select_decomp_table +from torch._inductor.compile_fx import compile_fx_inner, select_decomp_table from torch._inductor.pattern_matcher import PatternMatcherPass from torch._subclasses import FakeTensor from torch.fx import GraphModule @@ -153,7 +153,7 @@ def optimize( self.generate_events(num_events) return gm elif self.enable_inductor: - return compile_fx(gm, example_inputs) + return compile_fx_inner(gm, example_inputs) else: return gm diff --git a/tensorrt_llm/_torch/compilation/piecewise_optimizer.py b/tensorrt_llm/_torch/compilation/piecewise_optimizer.py index d4172201eb13..ceaa657c8fb3 100644 --- a/tensorrt_llm/_torch/compilation/piecewise_optimizer.py +++ b/tensorrt_llm/_torch/compilation/piecewise_optimizer.py @@ -4,7 +4,7 @@ import torch from torch._guards import detect_fake_mode -from torch._inductor.compile_fx import compile_fx +from torch._inductor.compile_fx import compile_fx, compile_fx_inner from torch._subclasses import FakeTensor from torch.fx import GraphModule, Interpreter from torch.fx.passes.split_module import split_module @@ -96,7 +96,8 @@ def call_module(self, target, args, kwargs): runtime_num_tokens_idx, self.capture_num_tokens, self.graph_pool_handle, - compile_fx(submod, args) if self.enable_inductor else submod, + compile_fx_inner(submod, args) + if self.enable_inductor else submod, self.enable_inductor, self.piecewise_runner_idx == 0, self.piecewise_runner_idx == self.piecewise_runner_num - 1, diff --git a/tests/integration/defs/verl/verl_config.yml b/tests/integration/defs/verl/verl_config.yml index db43920203de..286c59428670 100644 --- a/tests/integration/defs/verl/verl_config.yml +++ b/tests/integration/defs/verl/verl_config.yml @@ -15,8 +15,8 @@ verl_config: - "pip install nvidia-nvshmem-cu13==3.3.20" # Create nvshmem symlink (needed before DeepEP build) - >- - (cd /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib && - ln -s libnvshmem_host.so.3 libnvshmem_host.so) + (cd /usr/local/cuda/targets/x86_64-linux/lib && + [ -e libnvshmem_host.so ] || ln -s libnvshmem_host.so.3 libnvshmem_host.so) # Install DeepEP - >- git clone -b hybrid-ep https://github.com/deepseek-ai/DeepEP.git && @@ -36,7 +36,7 @@ verl_config: # The environment variables to expose in the container before setting up env_vars: - - "NVSHMEM_DIR=/usr/local/lib/python3.12/dist-packages/nvidia/nvshmem" + - "NVSHMEM_DIR=/usr/local/cuda/targets/x86_64-linux" - "LD_LIBRARY_PATH=\"${NVSHMEM_DIR}/lib:$LD_LIBRARY_PATH\"" - "PATH=\"${NVSHMEM_DIR}/bin:$PATH\"" - "TRTLLM_TEST_MODEL_PATH_ROOT=/tmp/verl-models" diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py index bfb79a4d3e37..82651348ea0a 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py @@ -574,7 +574,7 @@ def break_fp4_bytes(a, dtype): # Scale the input and weights to prevent large absolute values. FP4_X_GEN_SCALE = 0.5 -FP4_W_GEN_SCALE = 0.1 +FP4_W_GEN_SCALE = 0.75 @pytest.mark.parametrize("batch_size", BATCH_SIZES)