[TRTLLMINF-10][chore] move repeated apt-get installs into tritondevel Docker …

docker/Dockerfile.multi

@@ -31,62 +31,38 @@ FROM base AS devel
 # NB: PyTorch requires this to be < 1.0
 ENV PYTORCH_ALLOC_CONF="garbage_collection_threshold:0.99999"
 
-# Copy all installation scripts at once to reduce layers
-COPY docker/common/install.sh \
-     docker/common/install_base.sh \
-     docker/common/install_cmake.sh \
-     docker/common/install_ccache.sh \
-     docker/common/install_cuda_toolkit.sh \
-     docker/common/install_tensorrt.sh \
-     docker/common/install_polygraphy.sh \
-     docker/common/install_mpi4py.sh \
-     docker/common/install_pytorch.sh \
-     docker/common/install_ucx.sh \
-     docker/common/install_nixl.sh \
-     docker/common/install_etcd.sh \
-     ./
-
 ARG TRT_VER
 ARG CUDA_VER
 ARG CUDNN_VER
 ARG NCCL_VER
 ARG CUBLAS_VER
 ARG TORCH_INSTALL_TYPE="skip"
-RUN GITHUB_MIRROR=${GITHUB_MIRROR} \
+RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \
+    --mount=type=cache,target=/root/.cache/pip \
+    echo "Using GitHub mirror: ${GITHUB_MIRROR}" && \
+    echo "Using Python version: ${PYTHON_VERSION}" && \
+    GITHUB_MIRROR=${GITHUB_MIRROR} \
     PYTHON_VERSION=${PYTHON_VERSION} \
-    TRT_VER=${TRT_VER} \
-    CUDA_VER=${CUDA_VER} \
-    CUDNN_VER=${CUDNN_VER} \
-    NCCL_VER=${NCCL_VER} \
-    CUBLAS_VER=${CUBLAS_VER} \
+    TRT_VER=${TRT_VER} CUDA_VER=${CUDA_VER} CUDNN_VER=${CUDNN_VER} \
+    NCCL_VER=${NCCL_VER} CUBLAS_VER=${CUBLAS_VER} \
     TORCH_INSTALL_TYPE=${TORCH_INSTALL_TYPE} \
-    bash ./install.sh --base --cmake --ccache --cuda_toolkit --tensorrt --polygraphy --mpi4py --pytorch --opencv && \
-    rm install_base.sh && \
-    rm install_cmake.sh && \
-    rm install_ccache.sh && \
-    rm install_cuda_toolkit.sh && \
-    rm install_tensorrt.sh && \
-    rm install_polygraphy.sh && \
-    rm install_mpi4py.sh && \
-    rm install_pytorch.sh && \
-    rm install.sh
-
-# Copy and install dependencies from constraints.txt
-COPY constraints.txt /tmp/constraints.txt
-RUN pip3 install --no-cache-dir -r /tmp/constraints.txt && rm /tmp/constraints.txt
+    bash /opt/docker/common/install.sh --base --cmake --ccache --cuda_toolkit \
+         --tensorrt --polygraphy --mpi4py --pytorch --opencv
 
-# Remove nbconvert to avoid https://github.com/advisories/GHSA-xm59-rqc7-hhvf in the base NGC PyTorch image.
-RUN pip3 uninstall -y nbconvert || true
+# Install constraints after install.sh so cleanup() doesn't delete the file mid-RUN
+COPY constraints.txt /tmp/constraints.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip3 install --no-cache-dir -r /tmp/constraints.txt && \
+    rm /tmp/constraints.txt && \
+    pip3 uninstall -y nbconvert || true
 
 # Install UCX, NIXL, etcd
 # TODO: Combine these into the main install.sh script
-RUN GITHUB_MIRROR=${GITHUB_MIRROR} bash ./install_ucx.sh && \
-    GITHUB_MIRROR=${GITHUB_MIRROR} bash ./install_nixl.sh && \
-    bash ./install_etcd.sh && \
-    rm install_ucx.sh && \
-    rm install_nixl.sh && \
-    rm install_etcd.sh && \
-    rm -rf /root/.cache/pip && \
+RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \
+    --mount=type=cache,target=/root/.cache/pip \
+    GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \
+    GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \
+    bash /opt/docker/common/install_etcd.sh && \
     rm -rf /root/.cache/uv/archive-v0 && \
     # WAR against https://github.com/advisories/GHSA-58pv-8j8x-9vj2
     rm -rf /usr/local/lib/python3.12/dist-packages/setuptools/_vendor/jaraco.context-5.3.0.dist-info && \
@@ -96,9 +72,8 @@ RUN GITHUB_MIRROR=${GITHUB_MIRROR} bash ./install_ucx.sh && \
 # Generate OSS attribution file for devel image
 ARG TRT_LLM_VER
 ARG TARGETARCH
-COPY scripts/generate_container_oss_attribution.sh /tmp/generate_container_oss_attribution.sh
-RUN bash /tmp/generate_container_oss_attribution.sh "devel" "${TRT_LLM_VER}" "${TARGETARCH}" && \
-    rm /tmp/generate_container_oss_attribution.sh
+RUN --mount=type=bind,source=scripts/generate_container_oss_attribution.sh,target=/tmp/gen_attribution.sh \
+    bash /tmp/gen_attribution.sh "devel" "${TRT_LLM_VER}" "${TARGETARCH}"
 
 FROM ${TRITON_IMAGE}:${TRITON_BASE_TAG} AS triton
 
@@ -111,20 +86,15 @@ COPY --from=triton /opt/tritonserver/include /opt/tritonserver/include
 COPY --from=triton /opt/tritonserver/bin /opt/tritonserver/bin
 COPY --from=triton /opt/tritonserver/caches /opt/tritonserver/caches
 
-# Copy all installation scripts at once to reduce layers
-COPY docker/common/install_triton.sh \
-     docker/common/install_mooncake.sh \
-     ./
-
-# Install Mooncake, after triton handles boost requirement
-RUN GITHUB_MIRROR=${GITHUB_MIRROR} bash ./install_triton.sh && \
+# Install Triton deps, Mooncake, and CI tooling packages
+RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \
+    GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_triton.sh && \
     if [ -f /etc/redhat-release ]; then \
         echo "Rocky8 detected, skipping mooncake installation"; \
     else \
-        bash ./install_mooncake.sh; \
+        bash /opt/docker/common/install_mooncake.sh; \
     fi && \
-    rm install_triton.sh && \
-    rm install_mooncake.sh
+    bash /opt/docker/common/install_ci.sh
 
 FROM ${DEVEL_IMAGE} AS wheel
 WORKDIR /src/tensorrt_llm
@@ -136,8 +106,6 @@ COPY tensorrt_llm tensorrt_llm
 COPY 3rdparty 3rdparty
 COPY .gitmodules setup.py requirements.txt requirements-dev.txt constraints.txt README.md ./
 
-# Create cache directories for pip and ccache
-RUN mkdir -p /root/.cache/pip /root/.cache/ccache
 ENV CCACHE_DIR=/root/.cache/ccache
 # Build the TRT-LLM wheel
 ARG GITHUB_MIRROR=""
@@ -148,9 +116,6 @@ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACH
 
 FROM ${DEVEL_IMAGE} AS release
 
-# Create a cache directory for pip
-RUN mkdir -p /root/.cache/pip
-
 WORKDIR /app/tensorrt_llm
 RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,from=wheel,source=/src/tensorrt_llm/build,target=/tmp/wheel \
     pip install /tmp/wheel/tensorrt_llm*.whl
@@ -184,7 +149,6 @@ RUN chmod -R a+w examples && \
       benchmarks/cpp/gptManagerBenchmark.cpp \
       benchmarks/cpp/disaggServerBenchmark.cpp \
       benchmarks/cpp/CMakeLists.txt && \
-    rm -rf /root/.cache/pip && \
     rm -rf /root/.cache/uv/archive-v0 && \
     # WAR against https://github.com/advisories/GHSA-58pv-8j8x-9vj2
     rm -rf /usr/local/lib/python3.12/dist-packages/setuptools/_vendor/jaraco.context-5.3.0.dist-info && \

docker/common/install_base.sh

@@ -72,7 +72,6 @@ init_ubuntu() {
     lld \
     llvm \
     libclang-rt-dev \
-    libffi-dev \
     libstdc++-14-dev \
     libnuma1 \
     libnuma-dev \
@@ -148,7 +147,6 @@ install_gcctoolset_rockylinux() {
     wget \
     git-lfs \
     gcc-toolset-11 \
-    libffi-devel \
     -y
   dnf install \
     openmpi \

docker/common/install_ci.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+install_ci_ubuntu() {
+  apt-get update && apt-get install -y --no-install-recommends \
+    autossh \
+    libffi-dev \
+    openssh-client \
+    openssh-server \
+    rsync \
+    sshpass \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+}
+
+install_ci_rocky() {
+  dnf install -y \
+    autossh \
+    libffi-devel \
+    openssh-clients \
+    openssh-server \
+    rsync \
+    sshpass
+}
+
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu) install_ci_ubuntu ;;
+  rocky)  install_ci_rocky  ;;
+  *) echo "Unsupported OS: $ID"; exit 1 ;;
+esac

docker/common/install_mpi4py.sh

@@ -83,4 +83,3 @@ pip3 install --no-cache-dir "$TMP_DIR/mpi4py-${MPI4PY_VERSION}"
 
 # Clean up
 rm -rf "$TMP_DIR"
-rm -rf ~/.cache/pip

docker/common/install_polygraphy.sh

@@ -6,8 +6,3 @@ if [ -n "${GITHUB_MIRROR}" ]; then
   export PIP_INDEX_URL="https://urm.nvidia.com/artifactory/api/pypi/pypi-remote/simple"
 fi
 pip3 install polygraphy==0.49.26
-
-# Clean up pip cache and temporary files
-pip3 cache purge
-rm -rf ~/.cache/pip
-rm -rf /tmp/*

jenkins/L0_Test.groovy

@@ -434,8 +434,6 @@ def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName
     CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote ->
         def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}"
 
-        Utils.exec(pipeline, script: "apt-get update && apt-get install -y sshpass openssh-client")
-
         Utils.exec(pipeline, script: "echo Sleeping to allow Slurm job completion; sleep 30")
 
         def slurmJobID = Utils.exec(
@@ -489,8 +487,6 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName,
 
     Utils.exec(pipeline, script: "echo Sleeping to allow node destruction; sleep 30")
 
-    Utils.exec(pipeline, script: "apt-get update && apt-get install -y sshpass openssh-client")
-
     CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote ->
         Utils.exec(pipeline, script: "echo Slurm job ID: ${slurmJobID}")
 
@@ -1947,7 +1943,6 @@ def launchTestListCheck(pipeline)
     trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(LLM_DOCKER_IMAGE, "a10"), "trt-llm", {
         try {
             echoNodeAndGpuInfo(pipeline, stageName)
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y libffi-dev")
             sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m"
             // download TRT-LLM tarfile
             def tarName = BUILD_CONFIGS[VANILLA_CONFIG][TARNAME]
@@ -2571,15 +2566,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         // setup HF_HOME to cache model and datasets
         // init the huggingface cache from nfs, since the nfs is read-only, and HF_HOME needs to be writable, otherwise it will fail at creating file lock
         sh "mkdir -p ${HF_HOME} && ls -alh ${HF_HOME}"
-        trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y rsync")
         trtllm_utils.llmExecStepWithRetry(pipeline, script: "rsync -r ${MODEL_CACHE_DIR}/hugging-face-cache/ ${HF_HOME}/ && ls -lh ${HF_HOME}")
         sh "df -h"
 
         // install package
         sh "env | sort"
         sh "which python3"
         sh "python3 --version"
-        trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y libffi-dev")
+
         sh "rm -rf results-${stageName}.tar.gz ${stageName}/*"
         // download TRT-LLM tarfile
         def tarName = BUILD_CONFIGS[config][TARNAME]
@@ -2613,10 +2607,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         {
             testFilter[(DEBUG_MODE)] = false
 
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install openssh-server -y")
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install autossh -y")
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install sshpass -y")
-
             sh """
                 echo 'Port 22' >> /etc/ssh/sshd_config
                 echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config

jenkins/current_image_tags.properties

@@ -13,7 +13,7 @@
 #     images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead.
 IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm
 
-LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.12-py3-x86_64-ubuntu24.04-trt10.14.1.48-skip-tritondevel-202603051044-11898
-LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.12-py3-aarch64-ubuntu24.04-trt10.14.1.48-skip-tritondevel-202603051044-11898
-LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py310-trt10.14.1.48-skip-tritondevel-202603051044-11898
-LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py312-trt10.14.1.48-skip-tritondevel-202603051044-11898
+LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.12-py3-x86_64-ubuntu24.04-trt10.14.1.48-skip-tritondevel-202603122224-11720
+LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.12-py3-aarch64-ubuntu24.04-trt10.14.1.48-skip-tritondevel-202603122224-11720
+LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py310-trt10.14.1.48-skip-tritondevel-202603122224-11720
+LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py312-trt10.14.1.48-skip-tritondevel-202603122224-11720