sccache: integrate with Velox build

README.md

@@ -21,6 +21,23 @@ A Docker-based build infrastructure has been added to facilitate building Velox
 
 Specifically, the `velox-testing` and `velox` repositories must be checked out as sibling directories under the same parent directory. Once that is done, navigate (`cd`) into the `velox-testing/velox/scripts` directory and execute the build script `build_velox.sh`. After a successful build, the Velox libraries and executables are available in the container at `/opt/velox-build/release`.
 
+## `sccache` Usage
+`sccache` has been integrated to significantly accelerate builds by using a build server to share cached compiled objects during the build process. Currently it is only supported for Velox builds (not Presto.)
+
+### Setup and Usage
+First, set up authentication credentials:
+```bash
+cd velox-testing/velox/scripts
+./setup_sccache_auth.sh
+```
+
+Then build Velox with sccache enabled:
+```bash
+./build_velox.sh --sccache --sccache-auth-dir ~/.sccache-auth
+```
+
+The authentication setup only needs to be done once and credentials are valid for 12 hours.
+
 ## Velox Benchmarking
 A Docker-based benchmarking infrastructure has been added to facilitate running Velox benchmarks with support for CPU/GPU execution engines and profiling capabilities. The infrastructure uses a dedicated `velox-benchmark` Docker service with pre-configured volume mounts that automatically sync benchmark data and results. The data follows Hive directory structure, making it compatible with Presto. Currently, only TPC-H is implemented, but the infrastructure is designed to be easily extended to support additional benchmarks in the future.
 

velox/docker/adapters_build.dockerfile

@@ -10,6 +10,7 @@ ARG BUILD_WITH_VELOX_ENABLE_WAVE=OFF
 ARG TREAT_WARNINGS_AS_ERRORS=1
 ARG VELOX_ENABLE_BENCHMARKS=ON
 ARG BUILD_BASE_DIR=/opt/velox-build
+ARG ENABLE_SCCACHE=OFF
 
 # Environment mirroring upstream CI defaults and incorporating build args
 ENV VELOX_DEPENDENCY_SOURCE=SYSTEM \
@@ -41,13 +42,27 @@ ${BUILD_BASE_DIR}/release/_deps/cudf-build:\
 ${BUILD_BASE_DIR}/release/_deps/rmm-build:\
 ${BUILD_BASE_DIR}/release/_deps/rapids_logger-build:\
 ${BUILD_BASE_DIR}/release/_deps/kvikio-build:\
-${BUILD_BASE_DIR}/release/_deps/nvcomp_proprietary_binary-src/lib64"
+${BUILD_BASE_DIR}/release/_deps/nvcomp_proprietary_binary-src/lib64" \
+    ENABLE_SCCACHE=${ENABLE_SCCACHE}
 
 WORKDIR /workspace/velox
 
 # Print environment variables for debugging
 RUN printenv | sort
 
+# Install sccache if enabled
+RUN if [ "$ENABLE_SCCACHE" = "ON" ]; then \
+      set -euxo pipefail && \
+      # Install RAPIDS sccache fork
+      wget --no-hsts -q -O- "https://github.com/rapidsai/sccache/releases/download/v0.10.0-rapids.68/sccache-v0.10.0-rapids.68-$(uname -m)-unknown-linux-musl.tar.gz" | \
+      tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' 2>/dev/null && \
+      chmod +x /usr/bin/sccache && \
+      # Verify installation
+      sccache --version; \
+    else \
+      echo "Skipping sccache installation (ENABLE_SCCACHE=OFF)"; \
+    fi
+
 # Install NVIDIA Nsight Systems (nsys) for profiling - only if benchmarks are enabled
 RUN if [ "$VELOX_ENABLE_BENCHMARKS" = "ON" ]; then \
       set -euxo pipefail && \
@@ -64,7 +79,30 @@ RUN if [ "$VELOX_ENABLE_BENCHMARKS" = "ON" ]; then \
       echo "Skipping nsys installation (VELOX_ENABLE_BENCHMARKS=OFF)"; \
     fi
 
+# Copy sccache setup script (if sccache enabled)
+COPY velox-testing/velox/docker/sccache/sccache_setup.sh /sccache_setup.sh
+RUN if [ "$ENABLE_SCCACHE" = "ON" ]; then chmod +x /sccache_setup.sh; fi
+
+# Copy sccache auth files (note source of copy must be within the docker build context)
+COPY velox-testing/velox/docker/sccache/sccache_auth/ /sccache_auth/
+
 # Build in Release mode into ${BUILD_BASE_DIR}
 RUN --mount=type=bind,source=velox,target=/workspace/velox,ro \
     set -euxo pipefail && \
-    make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" BUILD_BASE_DIR="${BUILD_BASE_DIR}"
+    # Configure sccache if enabled
+    if [ "$ENABLE_SCCACHE" = "ON" ]; then \
+      # Run sccache setup script
+      /sccache_setup.sh && \
+      # Add sccache CMake flags
+      EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache" && \
+      echo "sccache distributed status:" && \
+      sccache --dist-status && \
+      echo "Pre-build sccache (zeroed out) statistics:" && \
+      sccache --show-stats; \
+    fi && \
+    make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS}" BUILD_BASE_DIR="${BUILD_BASE_DIR}"; \
+    # Show final sccache stats if enabled
+    if [ "$ENABLE_SCCACHE" = "ON" ]; then \
+      echo "Post-build sccache statistics:" && \
+      sccache --show-stats; \
+    fi

velox/docker/sccache/sccache_setup.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Check for required auth files
+if [[ ! -f /sccache_auth/github_token ]]; then
+    echo "ERROR: GitHub token not found at /sccache_auth/github_token"
+    exit 1
+fi
+
+if [[ ! -f /sccache_auth/aws_credentials ]]; then
+    echo "ERROR: AWS credentials not found at /sccache_auth/aws_credentials"
+    exit 1
+fi
+
+# Set up directories
+mkdir -p ~/.config/sccache ~/.aws
+
+# Install AWS credentials
+cp /sccache_auth/aws_credentials ~/.aws/credentials
+
+# Verify AWS credentials file exists and has content
+if [[ ! -s ~/.aws/credentials ]]; then
+    echo "ERROR: AWS credentials file is empty or invalid"
+    exit 1
+fi
+
+echo "AWS credentials file preview:"
+head -3 ~/.aws/credentials
+
+# Set AWS environment variables to ensure sccache uses the credentials file
+export AWS_SHARED_CREDENTIALS_FILE=~/.aws/credentials
+export AWS_PROFILE=default
+
+# Read GitHub token
+GITHUB_TOKEN=$(cat /sccache_auth/github_token | tr -d '\n\r ')
+
+# Create sccache config
+SCCACHE_ARCH=$(uname -m | sed 's/x86_64/amd64/')
+
+cat > ~/.config/sccache/config << SCCACHE_EOF
+[cache.disk]
+size = 107374182400
+
+[cache.disk.preprocessor_cache_mode]
+use_preprocessor_cache_mode = true
+
+[cache.s3]
+bucket = "rapids-sccache-devs"
+region = "us-east-2"
+no_credentials = false
+
+[dist]
+scheduler_url = "https://${SCCACHE_ARCH}.linux.sccache.rapids.nvidia.com"
+
+[dist.auth]
+type = "token"
+token = "${GITHUB_TOKEN}"
+SCCACHE_EOF
+
+# Configure sccache for high parallelism
+# Increase file descriptor limit for high parallelism (if possible)
+ulimit -n $(ulimit -Hn) || echo "Could not increase file descriptor limit"
+
+# Start sccache server
+sccache --start-server
+
+# Test sccache 
+sccache --show-stats
+
+# Testing distributed compilation status
+if sccache --dist-status; then
+    echo "Distributed compilation is available"
+else
+    echo "Error: Distributed compilation not available, check connectivity"
+    exit 1
+fi 

velox/scripts/build_velox.sh

@@ -11,6 +11,8 @@ VELOX_ENABLE_BENCHMARKS="ON"
 LOG_ENABLED=false
 TREAT_WARNINGS_AS_ERRORS="${TREAT_WARNINGS_AS_ERRORS:-1}"
 LOGFILE="./build_velox.log"
+ENABLE_SCCACHE=false
+SCCACHE_AUTH_DIR=""
 
 print_help() {
   cat <<EOF
@@ -27,8 +29,23 @@ Options:
   --gpu                       Build with GPU support (enables CUDF; sets BUILD_WITH_VELOX_ENABLE_CUDF=ON) [default].
   -j|--num-threads            NUM Number of threads to use for building (default: 3/4 of CPU cores).
   --benchmarks true|false     Enable benchmarks and nsys profiling tools (default: true).
+  --sccache                   Enable sccache distributed compilation caching.
+  --sccache-auth-dir DIR      Directory containing sccache authentication files (github_token, aws_credentials).
   -h, --help                  Show this help message and exit.
 
+sccache Usage:
+  The --sccache option enables distributed compilation caching using the RAPIDS sccache infrastructure.
+
+  Setup authentication first:
+    ./setup_sccache_auth.sh [output_dir]
+
+  Then use with build:
+    --sccache --sccache-auth-dir ~/.sccache-auth
+
+  Or set environment variable:
+    export SCCACHE_AUTH_DIR=~/.sccache-auth
+    ./build_velox.sh --sccache
+
 Examples:
   $(basename "$0") --all-cuda-archs --no-cache
   $(basename "$0") --plain
@@ -40,6 +57,9 @@ Examples:
   $(basename "$0") --log mybuild.log --all-cuda-archs
   $(basename "$0") -j 8 --gpu
   $(basename "$0") --num-threads 16 --no-cache
+  $(basename "$0") --sccache                                    # Local cache only
+  $(basename "$0") --sccache --sccache-github-token ghp_xxx  # Full distributed compilation
+  $(basename "$0") --sccache --sccache-show-stats            # Build with sccache and show stats
 
 By default, the script builds for the Native CUDA architecture (detected on host), uses Docker cache, standard build output, GPU support (CUDF enabled), and benchmarks enabled.
 EOF
@@ -107,6 +127,18 @@ parse_args() {
           exit 1
         fi
         ;;
+      --sccache)
+        ENABLE_SCCACHE=true
+        shift
+        ;;
+      --sccache-auth-dir)
+        if [[ -z "${2:-}" || "${2}" =~ ^- ]]; then
+          echo "Error: --sccache-auth-dir requires a directory path"
+          exit 1
+        fi
+        SCCACHE_AUTH_DIR="$2"
+        shift 2
+        ;;
       -h|--help)
         print_help
         exit 0
@@ -120,6 +152,30 @@ parse_args() {
   done
 }
 
+# Validate sccache authentication
+validate_sccache_auth() {
+  if [[ "$ENABLE_SCCACHE" == true ]]; then
+
+    if [[ -n "$SCCACHE_AUTH_DIR" ]]; then
+      if [[ ! -d "$SCCACHE_AUTH_DIR" ]]; then
+        echo "ERROR: sccache auth directory not found: $SCCACHE_AUTH_DIR"
+        exit 1
+      fi
+      if [[ ! -f "$SCCACHE_AUTH_DIR/github_token" ]]; then
+        echo "ERROR: GitHub token not found: $SCCACHE_AUTH_DIR/github_token"
+        exit 1
+      fi
+      if [[ ! -f "$SCCACHE_AUTH_DIR/aws_credentials" ]]; then
+        echo "ERROR: AWS credentials not found: $SCCACHE_AUTH_DIR/aws_credentials"
+        exit 1
+      fi
+      echo "sccache authentication files found in: $SCCACHE_AUTH_DIR"
+    else
+      echo "ERROR: No sccache auth directory provided but sccache is enabled. Run setup_sccache_auth.sh first."
+      exit 1
+    fi
+  fi
+}
 
 # Detect CUDA architecture since native architecture detection doesn't work
 # inside Docker containers
@@ -138,10 +194,11 @@ detect_cuda_architecture() {
   fi
 }
 
-
-
 parse_args "$@"
 
+# Validate sccache authentication if sccache is enabled
+validate_sccache_auth
+
 # Validate repo layout using shared script
 ../../scripts/validate_directories_exist.sh "../../../velox"
 
@@ -164,6 +221,17 @@ DOCKER_BUILD_OPTS+=(--build-arg NUM_THREADS="${NUM_THREADS}")
 DOCKER_BUILD_OPTS+=(--build-arg VELOX_ENABLE_BENCHMARKS="${VELOX_ENABLE_BENCHMARKS}")
 DOCKER_BUILD_OPTS+=(--build-arg TREAT_WARNINGS_AS_ERRORS="${TREAT_WARNINGS_AS_ERRORS}")
 
+# Add sccache build arguments
+if [[ "$ENABLE_SCCACHE" == true ]]; then
+  DOCKER_BUILD_OPTS+=(--build-arg ENABLE_SCCACHE="ON")
+  # Copy auth files to build context
+  mkdir -p ../docker/sccache/sccache_auth/
+  cp "$SCCACHE_AUTH_DIR/github_token" ../docker/sccache/sccache_auth/
+  cp "$SCCACHE_AUTH_DIR/aws_credentials" ../docker/sccache/sccache_auth/
+else
+  DOCKER_BUILD_OPTS+=(--build-arg ENABLE_SCCACHE="OFF")
+fi
+
 if [[ "$LOG_ENABLED" == true ]]; then
   echo "Logging build output to $LOGFILE"
   docker compose -f "$COMPOSE_FILE" build "${DOCKER_BUILD_OPTS[@]}" | tee "$LOGFILE"
@@ -173,6 +241,7 @@ else
   BUILD_EXIT_CODE=$?
 fi
 
+
 if [[ "$BUILD_EXIT_CODE" == "0" ]]; then
   if docker compose  -f "$COMPOSE_FILE" run --rm "${CONTAINER_NAME}" test -d "${EXPECTED_OUTPUT_DIR}" 2>/dev/null; then
     echo "  Built velox-adapters. View logs with:"
@@ -189,6 +258,13 @@ if [[ "$BUILD_EXIT_CODE" == "0" ]]; then
     else
       echo "  Benchmarks and nsys profiling are disabled in this build."
     fi
+    if [[ "$ENABLE_SCCACHE" == true ]]; then
+      echo "  sccache distributed compilation caching was enabled for this build."
+      if [[ -n "$SCCACHE_AUTH_DIR" ]]; then
+        echo "  To check sccache stats, run:"
+        echo "    docker compose -f $COMPOSE_FILE run --rm ${CONTAINER_NAME} sccache --show-stats"
+      fi
+    fi
     echo ""
   else
     echo "  ERROR: Build succeeded but ${EXPECTED_OUTPUT_DIR} not found in the container."