Alexey-Rivkin · Sep 28, 2026
@@ -128,7 +128,7 @@ steps:
     containerSelector: "{name: 'podman'}"
     run: |
       # Clone UCX source for nixlbench
-      git clone https://github.com/openucx/ucx.git ucx-src
+      .ci/scripts/retry.sh git clone https://github.com/openucx/ucx.git ucx-src
       git -C ucx-src checkout "${UCX_SHA}"
 
       "benchmark/nixlbench/contrib/build.sh" \

@@ -72,7 +72,7 @@ steps:
       # nixlbench builds against a UCX source tree; nixl consumes UCX_VERSION directly.
       if [ "$target" = "nixlbench" ]; then
         rm -rf ucx-src
-        git clone https://github.com/openucx/ucx.git ucx-src
+        .ci/scripts/retry.sh git clone https://github.com/openucx/ucx.git ucx-src
         git -C ucx-src checkout "$UCX_VERSION"
       fi
 

@@ -0,0 +1,22 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs a command up to 5 times with growing backoff; used for github.com fetches in the CI builds.
+for i in 1 2 3 4 5; do
+    "$@" && exit 0
+    [ "$i" -lt 5 ] && sleep $((i * 15))
+done
+exit 1
@@ -21,6 +21,10 @@ set -e
 set -x
 set -o pipefail
 
+# Retry helper: the github.com fetches below fail transiently under load.
+RETRY_SH=$(readlink -f "$(dirname "$0")/../.ci/scripts/retry.sh")
+retry() { "$RETRY_SH" "$@"; }
+
 # Force CMake to always copy files in install directives, rather than skip based on file modification timestamp.
 # File modification timestamp check in CMake uses 1 second resolution.
 # This causes problems for fast builds that install, patch then reinstall the same file, as the final install step
@@ -250,9 +254,9 @@ else
 
     ( \
       cd ${BUILD_TMP} && \
-      git clone https://github.com/abseil/abseil-cpp.git && \
+      retry git clone https://github.com/abseil/abseil-cpp.git && \
       cd abseil-cpp && \
-      git fetch --depth 1 origin "${ABSL_TAG}" && \
+      retry git fetch --depth 1 origin "${ABSL_TAG}" && \
       git checkout "${ABSL_TAG}" && \
       mkdir -p build && cd build && \
       cmake .. \
@@ -273,7 +277,11 @@ else
 
     ( \
       cd ${BUILD_TMP} && \
-      git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+      retry git clone -b "${GRPC_TAG}" --depth 1 https://github.com/grpc/grpc && \
+      retry git -C grpc submodule update --init --depth 1 \
+          third_party/protobuf third_party/cares/cares third_party/re2 \
+          third_party/xds third_party/envoy-api third_party/googleapis \
+          third_party/protoc-gen-validate third_party/opencensus-proto && \
       cd grpc && \
       mkdir -p cmake/build && \
       cd cmake/build && \
@@ -301,7 +309,7 @@ else
 
     ( \
       cd ${BUILD_TMP} && \
-      git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
+      retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
       cd etcd-cpp-apiv3 && \
       sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \
       mkdir build && cd build && \
@@ -321,7 +329,8 @@ else
 
     ( \
       cd ${BUILD_TMP} && \
-      git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+      retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+      retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \
       mkdir aws_sdk_build && \
       cd aws_sdk_build && \
       cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=/usr/local && \
@@ -333,7 +342,7 @@ else
 
     ( \
       cd ${BUILD_TMP} && \
-      git clone https://github.com/nvidia/gusli.git && \
+      retry git clone https://github.com/nvidia/gusli.git && \
       cd gusli && \
       $SUDO make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 && \
       $SUDO ldconfig && \
@@ -345,7 +354,7 @@ else
       cd ${BUILD_TMP} && \
       MOONCAKE_VERSION="${MOONCAKE_VERSION:-v0.3.10.post1}" && \
       echo "MOONCAKE_VERSION: ${MOONCAKE_VERSION}" && \
-      git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \
+      retry git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \
       cd Mooncake && \
       sed -i '/liburing-dev/d' dependencies.sh
       $SUDO bash dependencies.sh -y && \
@@ -365,7 +374,7 @@ else
 
     ( \
       cd ${BUILD_TMP} &&
-      git clone --depth 1 https://github.com/google/gtest-parallel.git &&
+      retry git clone --depth 1 https://github.com/google/gtest-parallel.git &&
       mkdir -p ${INSTALL_DIR}/bin &&
       cp ${BUILD_TMP}/gtest-parallel/* ${INSTALL_DIR}/bin/
     )
@@ -374,7 +383,7 @@ else
       cd ${BUILD_TMP} && \
       df -h && \
       curl -sL https://aka.ms/InstallAzureCLIDeb | $SUDO bash && \
-      git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch  azure-storage-blobs_12.15.0 && \
+      retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch  azure-storage-blobs_12.15.0 && \
       cd azure-sdk-for-cpp/ && \
       mkdir build && cd build && \
       AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \
@@ -392,7 +401,7 @@ else
     if $HAS_GPU && test -d "$CUDA_HOME"; then
        ( \
         cd ${BUILD_TMP} && \
-        git clone https://github.com/uccl-project/uccl.git && \
+        retry git clone https://github.com/uccl-project/uccl.git && \
         cd uccl && git checkout -q "${UCCL_COMMIT_SHA}" && \
         cd p2p && \
         make -j"$NPROC" && \
@@ -402,10 +411,11 @@ else
     else
         echo "No NVIDIA GPU(s) detected. Skipping UCCL installation."
     fi
-    git clone https://github.com/openucx/ucx.git ${BUILD_TMP}/ucx
+    retry git clone https://github.com/openucx/ucx.git ${BUILD_TMP}/ucx
     ( \
     cd ${BUILD_TMP}/ucx && \
     git checkout "${UCX_VERSION}" && \
+    retry git submodule update --init && \
     ./autogen.sh && \
     ./contrib/configure-release-mt \
             --prefix="${UCX_INSTALL_DIR}" \
@@ -453,6 +463,7 @@ else
         printf "[binaries]\npython = '%s'\n" "${NIXL_PYTHON}" > "${NIXL_PYTHON_NATIVE_FILE}"
         NIXL_PYTHON_ARGS=(--native-file "${NIXL_PYTHON_NATIVE_FILE}" -Dpython.install_env=venv)
     fi
+    retry meson subprojects download taskflow prometheus-cpp
     # shellcheck disable=SC2086
     meson setup "${NIXL_PYTHON_ARGS[@]}" ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
     ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install

@@ -37,6 +37,9 @@ RUN set -e; \
 # --- Stage 1: Common OS setup ---
 FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS os_setup_stage
 
+# Retry helper: the github.com fetches below fail transiently under load.
+COPY --from=nixl .ci/scripts/retry.sh /usr/local/bin/retry
+
 # Install them only when the base has a matching runtime and no dev files of
 # its own, so bases that are already correct or have no cuObject are untouched.
 COPY --from=cuobj_dev /cuobj /tmp/cuobj
@@ -149,6 +152,7 @@ RUN echo "INFO: Starting custom UCX build..." && \
     echo "INFO: Removing pre-existing UCX installations..." && \
     rm -rf /usr/lib/ucx /opt/hpcx/ucx && \
     cd /workspace/ucx && \
+    retry git submodule update --init && \
     ./autogen.sh && \
     echo "INFO: Building UCX..." && \
     ./contrib/configure-release --with-cuda=/usr/local/cuda \
@@ -184,9 +188,9 @@ ARG GRPC_TAG="v1.73.0"
 WORKDIR /workspace
 
 # Build Abseil from source (required for absl/log).
-RUN git clone https://github.com/abseil/abseil-cpp.git && \
+RUN retry git clone https://github.com/abseil/abseil-cpp.git && \
     cd abseil-cpp && \
-    git fetch --depth 1 origin ${ABSL_TAG} && \
+    retry git fetch --depth 1 origin ${ABSL_TAG} && \
     git checkout ${ABSL_TAG} && \
     mkdir -p build && cd build && \
     cmake .. \
@@ -200,8 +204,13 @@ RUN git clone https://github.com/abseil/abseil-cpp.git && \
     make install && \
     ldconfig
 
-# Build gRPC from source against /usr/local Abseil.
-RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+# Build gRPC against /usr/local Abseil. Only the submodules this cmake config uses:
+# protobuf, c-ares, re2 and the xds protos (SSL/Abseil/zlib are packages, tests and OTel off).
+RUN retry git clone -b ${GRPC_TAG} --depth 1 https://github.com/grpc/grpc && \
+    retry git -C grpc submodule update --init --depth 1 \
+        third_party/protobuf third_party/cares/cares third_party/re2 \
+        third_party/xds third_party/envoy-api third_party/googleapis \
+        third_party/protoc-gen-validate third_party/opencensus-proto && \
     cd grpc && \
     mkdir -p cmake/build && \
     cd cmake/build && \
@@ -245,7 +254,7 @@ RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \
     make install && \
     ldconfig
 
-RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
+RUN retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
     cd etcd-cpp-apiv3 && \
     sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \
     mkdir build && cd build && \
@@ -255,7 +264,8 @@ RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git &&
 # Install AWS SDK C++ dependencies and build
 RUN apt-get update && apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev hwloc libhwloc-dev
 
-RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+RUN retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+    retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \
     mkdir sdk_build && \
     cd sdk_build && \
     cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=20 && \
@@ -264,7 +274,7 @@ RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github
 
 # Install install Azure SDK for C++ dependencies and build
 RUN apt-get update && apt-get install -y libxml2-dev
-RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \
+RUN retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \
     cd azure-sdk-for-cpp/ && \
     mkdir build && cd build && \
     AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \
@@ -274,7 +284,7 @@ RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch
     cmake --install sdk/storage/azure-storage-blobs && \
     cmake --install sdk/identity
 
-RUN git clone https://github.com/nvidia/gusli.git && \
+RUN retry git clone https://github.com/nvidia/gusli.git && \
     cd gusli && \
     make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 CXX="g++ -std=c++20" && \
     cd ..
@@ -315,6 +325,7 @@ WORKDIR /workspace/nixl
 
 RUN rm -rf build && \
     mkdir build && \
+    retry meson subprojects download taskflow prometheus-cpp && \
     meson setup build --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE && \
     cd build && \
     ninja -j${NPROC:-$(nproc)} && \

@@ -157,10 +157,13 @@ WORKDIR /workspace
 # Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption.
 ENV CMAKE_INSTALL_ALWAYS=1
 
+# Retry helper: the github.com fetches below fail transiently under load.
+COPY .ci/scripts/retry.sh /usr/local/bin/retry
+
 # Build Abseil from source (required for absl/log).
-RUN git clone https://github.com/abseil/abseil-cpp.git && \
+RUN retry git clone https://github.com/abseil/abseil-cpp.git && \
     cd abseil-cpp && \
-    git fetch --depth 1 origin ${ABSL_TAG} && \
+    retry git fetch --depth 1 origin ${ABSL_TAG} && \
     git checkout ${ABSL_TAG} && \
     mkdir -p build && cd build && \
     cmake .. \
@@ -174,8 +177,13 @@ RUN git clone https://github.com/abseil/abseil-cpp.git && \
     make install && \
     ldconfig
 
-# Build gRPC from source against system/install-prefix Abseil.
-RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+# Build gRPC against the prefix Abseil. Only the submodules this cmake config uses:
+# protobuf, c-ares, re2 and the xds protos (SSL/Abseil/zlib are packages, tests and OTel off).
+RUN retry git clone -b ${GRPC_TAG} --depth 1 https://github.com/grpc/grpc && \
+    retry git -C grpc submodule update --init --depth 1 \
+        third_party/protobuf third_party/cares/cares third_party/re2 \
+        third_party/xds third_party/envoy-api third_party/googleapis \
+        third_party/protoc-gen-validate third_party/opencensus-proto && \
     cd grpc && \
     mkdir -p cmake/build && \
     cd cmake/build && \
@@ -198,24 +206,25 @@ RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules
     make install && \
     ldconfig
 
-RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
+RUN retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
     cd etcd-cpp-apiv3 && \
     sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \
     mkdir build && cd build && \
     cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH=/usr/local && \
     make -j${NPROC:-$(nproc)} && make install
 
-RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+RUN retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \
+    retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \
     mkdir aws_sdk_build && cd aws_sdk_build && \
     cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=20 && \
     make -j${NPROC:-$(nproc)} && make install
 
-RUN git clone https://github.com/nvidia/gusli.git && \
+RUN retry git clone https://github.com/nvidia/gusli.git && \
      cd gusli && \
      make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 CXX="g++ -std=c++20" && \
      cd ..
 
-RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \
+RUN retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \
     cd azure-sdk-for-cpp/ && \
     mkdir build && cd build && \
     AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \
@@ -247,10 +256,11 @@ RUN rm -rf /usr/lib/ucx
 RUN rm -rf /opt/hpcx/ucx
 
 RUN cd /usr/local/src && \
-     git clone "$UCX_REPO" ucx && \
+     retry git clone "$UCX_REPO" ucx && \
      cd ucx && \
      echo "=== Using UCX_REF=$UCX_REF ===" && \
      git checkout $UCX_REF && \
+     retry git submodule update --init && \
      if [ "$BUILD_NIXL_EP" = "true" ]; then EXPERIMENTAL_API_PARAM="--enable-experimental-api"; else EXPERIMENTAL_API_PARAM=""; fi && \
      ./autogen.sh && \
      if [ -n "$UCX_SONAME_SUFFIX" ]; then \
@@ -292,7 +302,7 @@ RUN cd /usr/local/src && \
      ldconfig
 
 RUN cd /tmp && \
-     git clone --depth 1 https://github.com/google/gtest-parallel.git && \
+     retry git clone --depth 1 https://github.com/google/gtest-parallel.git && \
      mkdir -p /usr/local/bin && \
      cp gtest-parallel/gtest-parallel gtest-parallel/gtest_parallel.py /usr/local/bin/
 ENV PATH=/usr/local/bin:$PATH
@@ -416,6 +426,7 @@ RUN rm -rf build && \
     else \
         NIXL_EP_FLAG=""; \
     fi && \
+    retry meson subprojects download taskflow prometheus-cpp && \
     meson setup -Ducx_path=$UCX_PREFIX -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH $NIXL_EP_FLAG build/ --prefix=$NIXL_PREFIX --buildtype=$BUILD_TYPE && \
     cd build && \
     ninja -j${NPROC:-$(nproc)} && \