diff --git a/.ci/jenkins/lib/build-container-matrix.yaml b/.ci/jenkins/lib/build-container-matrix.yaml index fd2f914461..b5dc628e5c 100644 --- a/.ci/jenkins/lib/build-container-matrix.yaml +++ b/.ci/jenkins/lib/build-container-matrix.yaml @@ -128,7 +128,7 @@ steps: containerSelector: "{name: 'podman'}" run: | # Clone UCX source for nixlbench - git clone https://github.com/openucx/ucx.git ucx-src + .ci/scripts/retry.sh git clone https://github.com/openucx/ucx.git ucx-src git -C ucx-src checkout "${UCX_SHA}" "benchmark/nixlbench/contrib/build.sh" \ diff --git a/.ci/jenkins/lib/build-container-pr-matrix.yaml b/.ci/jenkins/lib/build-container-pr-matrix.yaml index 9c16227802..79815db6ca 100644 --- a/.ci/jenkins/lib/build-container-pr-matrix.yaml +++ b/.ci/jenkins/lib/build-container-pr-matrix.yaml @@ -72,7 +72,7 @@ steps: # nixlbench builds against a UCX source tree; nixl consumes UCX_VERSION directly. if [ "$target" = "nixlbench" ]; then rm -rf ucx-src - git clone https://github.com/openucx/ucx.git ucx-src + .ci/scripts/retry.sh git clone https://github.com/openucx/ucx.git ucx-src git -C ucx-src checkout "$UCX_VERSION" fi diff --git a/.ci/scripts/retry.sh b/.ci/scripts/retry.sh new file mode 100755 index 0000000000..d23a665611 --- /dev/null +++ b/.ci/scripts/retry.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs a command up to 5 times with growing backoff; used for github.com fetches in the CI builds. +for i in 1 2 3 4 5; do + "$@" && exit 0 + [ "$i" -lt 5 ] && sleep $((i * 15)) +done +exit 1 diff --git a/.gitlab/build.sh b/.gitlab/build.sh index f19359cc67..3c5c860d80 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -21,6 +21,10 @@ set -e set -x set -o pipefail +# Retry helper: the github.com fetches below fail transiently under load. +RETRY_SH=$(readlink -f "$(dirname "$0")/../.ci/scripts/retry.sh") +retry() { "$RETRY_SH" "$@"; } + # Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. # File modification timestamp check in CMake uses 1 second resolution. # This causes problems for fast builds that install, patch then reinstall the same file, as the final install step @@ -250,9 +254,9 @@ else ( \ cd ${BUILD_TMP} && \ - git clone https://github.com/abseil/abseil-cpp.git && \ + retry git clone https://github.com/abseil/abseil-cpp.git && \ cd abseil-cpp && \ - git fetch --depth 1 origin "${ABSL_TAG}" && \ + retry git fetch --depth 1 origin "${ABSL_TAG}" && \ git checkout "${ABSL_TAG}" && \ mkdir -p build && cd build && \ cmake .. \ @@ -273,7 +277,11 @@ else ( \ cd ${BUILD_TMP} && \ - git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + retry git clone -b "${GRPC_TAG}" --depth 1 https://github.com/grpc/grpc && \ + retry git -C grpc submodule update --init --depth 1 \ + third_party/protobuf third_party/cares/cares third_party/re2 \ + third_party/xds third_party/envoy-api third_party/googleapis \ + third_party/protoc-gen-validate third_party/opencensus-proto && \ cd grpc && \ mkdir -p cmake/build && \ cd cmake/build && \ @@ -301,7 +309,7 @@ else ( \ cd ${BUILD_TMP} && \ - git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ cd etcd-cpp-apiv3 && \ sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ mkdir build && cd build && \ @@ -321,7 +329,8 @@ else ( \ cd ${BUILD_TMP} && \ - git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \ mkdir aws_sdk_build && \ cd aws_sdk_build && \ cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=/usr/local && \ @@ -333,7 +342,7 @@ else ( \ cd ${BUILD_TMP} && \ - git clone https://github.com/nvidia/gusli.git && \ + retry git clone https://github.com/nvidia/gusli.git && \ cd gusli && \ $SUDO make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 && \ $SUDO ldconfig && \ @@ -345,7 +354,7 @@ else cd ${BUILD_TMP} && \ MOONCAKE_VERSION="${MOONCAKE_VERSION:-v0.3.10.post1}" && \ echo "MOONCAKE_VERSION: ${MOONCAKE_VERSION}" && \ - git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \ + retry git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \ cd Mooncake && \ sed -i '/liburing-dev/d' dependencies.sh $SUDO bash dependencies.sh -y && \ @@ -365,7 +374,7 @@ else ( \ cd ${BUILD_TMP} && - git clone --depth 1 https://github.com/google/gtest-parallel.git && + retry git clone --depth 1 https://github.com/google/gtest-parallel.git && mkdir -p ${INSTALL_DIR}/bin && cp ${BUILD_TMP}/gtest-parallel/* ${INSTALL_DIR}/bin/ ) @@ -374,7 +383,7 @@ else cd ${BUILD_TMP} && \ df -h && \ curl -sL https://aka.ms/InstallAzureCLIDeb | $SUDO bash && \ - git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ cd azure-sdk-for-cpp/ && \ mkdir build && cd build && \ AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \ @@ -392,7 +401,7 @@ else if $HAS_GPU && test -d "$CUDA_HOME"; then ( \ cd ${BUILD_TMP} && \ - git clone https://github.com/uccl-project/uccl.git && \ + retry git clone https://github.com/uccl-project/uccl.git && \ cd uccl && git checkout -q "${UCCL_COMMIT_SHA}" && \ cd p2p && \ make -j"$NPROC" && \ @@ -402,10 +411,11 @@ else else echo "No NVIDIA GPU(s) detected. Skipping UCCL installation." fi - git clone https://github.com/openucx/ucx.git ${BUILD_TMP}/ucx + retry git clone https://github.com/openucx/ucx.git ${BUILD_TMP}/ucx ( \ cd ${BUILD_TMP}/ucx && \ git checkout "${UCX_VERSION}" && \ + retry git submodule update --init && \ ./autogen.sh && \ ./contrib/configure-release-mt \ --prefix="${UCX_INSTALL_DIR}" \ @@ -453,6 +463,7 @@ else printf "[binaries]\npython = '%s'\n" "${NIXL_PYTHON}" > "${NIXL_PYTHON_NATIVE_FILE}" NIXL_PYTHON_ARGS=(--native-file "${NIXL_PYTHON_NATIVE_FILE}" -Dpython.install_env=venv) fi + retry meson subprojects download taskflow prometheus-cpp # shellcheck disable=SC2086 meson setup "${NIXL_PYTHON_ARGS[@]}" ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index 6d188aa0b7..5661a9d62a 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -37,6 +37,9 @@ RUN set -e; \ # --- Stage 1: Common OS setup --- FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS os_setup_stage +# Retry helper: the github.com fetches below fail transiently under load. +COPY --from=nixl .ci/scripts/retry.sh /usr/local/bin/retry + # Install them only when the base has a matching runtime and no dev files of # its own, so bases that are already correct or have no cuObject are untouched. COPY --from=cuobj_dev /cuobj /tmp/cuobj @@ -149,6 +152,7 @@ RUN echo "INFO: Starting custom UCX build..." && \ echo "INFO: Removing pre-existing UCX installations..." && \ rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /workspace/ucx && \ + retry git submodule update --init && \ ./autogen.sh && \ echo "INFO: Building UCX..." && \ ./contrib/configure-release --with-cuda=/usr/local/cuda \ @@ -184,9 +188,9 @@ ARG GRPC_TAG="v1.73.0" WORKDIR /workspace # Build Abseil from source (required for absl/log). -RUN git clone https://github.com/abseil/abseil-cpp.git && \ +RUN retry git clone https://github.com/abseil/abseil-cpp.git && \ cd abseil-cpp && \ - git fetch --depth 1 origin ${ABSL_TAG} && \ + retry git fetch --depth 1 origin ${ABSL_TAG} && \ git checkout ${ABSL_TAG} && \ mkdir -p build && cd build && \ cmake .. \ @@ -200,8 +204,13 @@ RUN git clone https://github.com/abseil/abseil-cpp.git && \ make install && \ ldconfig -# Build gRPC from source against /usr/local Abseil. -RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ +# Build gRPC against /usr/local Abseil. Only the submodules this cmake config uses: +# protobuf, c-ares, re2 and the xds protos (SSL/Abseil/zlib are packages, tests and OTel off). +RUN retry git clone -b ${GRPC_TAG} --depth 1 https://github.com/grpc/grpc && \ + retry git -C grpc submodule update --init --depth 1 \ + third_party/protobuf third_party/cares/cares third_party/re2 \ + third_party/xds third_party/envoy-api third_party/googleapis \ + third_party/protoc-gen-validate third_party/opencensus-proto && \ cd grpc && \ mkdir -p cmake/build && \ cd cmake/build && \ @@ -245,7 +254,7 @@ RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ make install && \ ldconfig -RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ +RUN retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ cd etcd-cpp-apiv3 && \ sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ mkdir build && cd build && \ @@ -255,7 +264,8 @@ RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && # Install AWS SDK C++ dependencies and build RUN apt-get update && apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev hwloc libhwloc-dev -RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ +RUN retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \ mkdir sdk_build && \ cd sdk_build && \ cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=20 && \ @@ -264,7 +274,7 @@ RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github # Install install Azure SDK for C++ dependencies and build RUN apt-get update && apt-get install -y libxml2-dev -RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ +RUN retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ cd azure-sdk-for-cpp/ && \ mkdir build && cd build && \ AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \ @@ -274,7 +284,7 @@ RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch cmake --install sdk/storage/azure-storage-blobs && \ cmake --install sdk/identity -RUN git clone https://github.com/nvidia/gusli.git && \ +RUN retry git clone https://github.com/nvidia/gusli.git && \ cd gusli && \ make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 CXX="g++ -std=c++20" && \ cd .. @@ -315,6 +325,7 @@ WORKDIR /workspace/nixl RUN rm -rf build && \ mkdir build && \ + retry meson subprojects download taskflow prometheus-cpp && \ meson setup build --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE && \ cd build && \ ninja -j${NPROC:-$(nproc)} && \ diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 939cf54549..85eb1c2346 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -157,10 +157,13 @@ WORKDIR /workspace # Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. ENV CMAKE_INSTALL_ALWAYS=1 +# Retry helper: the github.com fetches below fail transiently under load. +COPY .ci/scripts/retry.sh /usr/local/bin/retry + # Build Abseil from source (required for absl/log). -RUN git clone https://github.com/abseil/abseil-cpp.git && \ +RUN retry git clone https://github.com/abseil/abseil-cpp.git && \ cd abseil-cpp && \ - git fetch --depth 1 origin ${ABSL_TAG} && \ + retry git fetch --depth 1 origin ${ABSL_TAG} && \ git checkout ${ABSL_TAG} && \ mkdir -p build && cd build && \ cmake .. \ @@ -174,8 +177,13 @@ RUN git clone https://github.com/abseil/abseil-cpp.git && \ make install && \ ldconfig -# Build gRPC from source against system/install-prefix Abseil. -RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ +# Build gRPC against the prefix Abseil. Only the submodules this cmake config uses: +# protobuf, c-ares, re2 and the xds protos (SSL/Abseil/zlib are packages, tests and OTel off). +RUN retry git clone -b ${GRPC_TAG} --depth 1 https://github.com/grpc/grpc && \ + retry git -C grpc submodule update --init --depth 1 \ + third_party/protobuf third_party/cares/cares third_party/re2 \ + third_party/xds third_party/envoy-api third_party/googleapis \ + third_party/protoc-gen-validate third_party/opencensus-proto && \ cd grpc && \ mkdir -p cmake/build && \ cd cmake/build && \ @@ -198,24 +206,25 @@ RUN git clone --recurse-submodules -b ${GRPC_TAG} --depth 1 --shallow-submodules make install && \ ldconfig -RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ +RUN retry git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ cd etcd-cpp-apiv3 && \ sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ mkdir build && cd build && \ cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH=/usr/local && \ make -j${NPROC:-$(nproc)} && make install -RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ +RUN retry git clone --depth 1 https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + retry git -C aws-sdk-cpp submodule update --init --recursive --depth 1 && \ mkdir aws_sdk_build && cd aws_sdk_build && \ cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=20 && \ make -j${NPROC:-$(nproc)} && make install -RUN git clone https://github.com/nvidia/gusli.git && \ +RUN retry git clone https://github.com/nvidia/gusli.git && \ cd gusli && \ make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=1 CXX="g++ -std=c++20" && \ cd .. -RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ +RUN retry git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ cd azure-sdk-for-cpp/ && \ mkdir build && cd build && \ AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \ @@ -247,10 +256,11 @@ RUN rm -rf /usr/lib/ucx RUN rm -rf /opt/hpcx/ucx RUN cd /usr/local/src && \ - git clone "$UCX_REPO" ucx && \ + retry git clone "$UCX_REPO" ucx && \ cd ucx && \ echo "=== Using UCX_REF=$UCX_REF ===" && \ git checkout $UCX_REF && \ + retry git submodule update --init && \ if [ "$BUILD_NIXL_EP" = "true" ]; then EXPERIMENTAL_API_PARAM="--enable-experimental-api"; else EXPERIMENTAL_API_PARAM=""; fi && \ ./autogen.sh && \ if [ -n "$UCX_SONAME_SUFFIX" ]; then \ @@ -292,7 +302,7 @@ RUN cd /usr/local/src && \ ldconfig RUN cd /tmp && \ - git clone --depth 1 https://github.com/google/gtest-parallel.git && \ + retry git clone --depth 1 https://github.com/google/gtest-parallel.git && \ mkdir -p /usr/local/bin && \ cp gtest-parallel/gtest-parallel gtest-parallel/gtest_parallel.py /usr/local/bin/ ENV PATH=/usr/local/bin:$PATH @@ -416,6 +426,7 @@ RUN rm -rf build && \ else \ NIXL_EP_FLAG=""; \ fi && \ + retry meson subprojects download taskflow prometheus-cpp && \ meson setup -Ducx_path=$UCX_PREFIX -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH $NIXL_EP_FLAG build/ --prefix=$NIXL_PREFIX --buildtype=$BUILD_TYPE && \ cd build && \ ninja -j${NPROC:-$(nproc)} && \