diff --git a/.ci/dockerfiles/Dockerfile.rocm b/.ci/dockerfiles/Dockerfile.rocm index 2b21b16c47..ce0aef4d1d 100644 --- a/.ci/dockerfiles/Dockerfile.rocm +++ b/.ci/dockerfiles/Dockerfile.rocm @@ -78,6 +78,14 @@ ENV PIP_CERT=${SYSTEM_CA_BUNDLE} \ # libgrpc_plugin_support.so from /opt/nixl/lib. Every stage inherits this ENV. ENV LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib64:/usr/local/lib:/usr/local/lib64" +# Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. +# File modification timestamp check in CMake uses 1 second resolution. +# This causes problems for fast builds that install, patch then reinstall the same file, as the final install step +# may be incorrectly skipped. +# Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. +# Inherited by every *-build stage and `final`. +ENV CMAKE_INSTALL_ALWAYS=1 + # Some ROCm base images ship broken cmake configs for these. RUN rm -rf /usr/lib/cmake/grpc /usr/lib/cmake/protobuf diff --git a/.gitlab/build-rocm.sh b/.gitlab/build-rocm.sh index d4ebfcc9ee..f565002381 100755 --- a/.gitlab/build-rocm.sh +++ b/.gitlab/build-rocm.sh @@ -23,6 +23,13 @@ set -e set -x set -o pipefail +# Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. +# File modification timestamp check in CMake uses 1 second resolution. +# This causes problems for fast builds that install, patch then reinstall the same file, as the final install step +# may be incorrectly skipped. +# Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. +export CMAKE_INSTALL_ALWAYS=1 + # Parse commandline arguments with first argument being the install directory # and second argument being the UCX installation directory. # Source dependency options grouped with dep download/build/install steps. diff --git a/.gitlab/build.sh b/.gitlab/build.sh index a15e9d0f0d..e261f1345f 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -21,6 +21,13 @@ set -e set -x set -o pipefail +# Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. +# File modification timestamp check in CMake uses 1 second resolution. +# This causes problems for fast builds that install, patch then reinstall the same file, as the final install step +# may be incorrectly skipped. +# Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. +export CMAKE_INSTALL_ALWAYS=1 + # Parse commandline arguments with first argument being the install directory # and second argument being the UCX installation directory. INSTALL_DIR=$1 diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 53f3c1bd2e..0ff91ed224 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -108,6 +108,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ WORKDIR /workspace +# Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. +# File modification timestamp check in CMake uses 1 second resolution. +# This causes problems for fast builds that install, patch then reinstall the same file, as the final install step +# may be incorrectly skipped. +# Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. +ENV CMAKE_INSTALL_ALWAYS=1 + # Build Abseil from source (required for absl/log). RUN git clone https://github.com/abseil/abseil-cpp.git && \ cd abseil-cpp && \ diff --git a/contrib/Dockerfile.manylinux b/contrib/Dockerfile.manylinux index aaea9a4d5e..955520a7cb 100644 --- a/contrib/Dockerfile.manylinux +++ b/contrib/Dockerfile.manylinux @@ -66,6 +66,15 @@ ENV PATH=/opt/rh/gcc-toolset-14/root/usr/bin:${PATH} \ # cmake_minimum_required(VERSION < 3.5). Some bundled deps (e.g. gRPC's c-ares) # still declare ancient minimums. This tells CMake 4.x to accept them. ENV CMAKE_POLICY_VERSION_MINIMUM=3.5 + +# Force CMake to always copy files in install directives, rather than skip based on file modification timestamp. +# File modification timestamp check in CMake uses 1 second resolution. +# This causes problems for fast builds that install, patch then reinstall the same file, as the final install step +# may be incorrectly skipped. +# Seen in CI as flaky ASAN failure due to inconsistent Azure SDK headers causing memory corruption. +# Inherited by the wheel stage below via FROM $wheel_base. +ENV CMAKE_INSTALL_ALWAYS=1 + # === end Option B header ====================================================== ARG LIBFABRIC_VERSION="v1.21.0" ARG HWLOC_VERSION="2.12.2"