[REVIEW][OPS] gpuCI Script Updates

CHANGELOG.md

@@ -103,8 +103,9 @@
 - PR #6273 Update JNI to use ORC options builder
 - PR #6293 Replace shuffle warp reduce with cub calls
 - PR #6287 Make java aggregate API follow C++ API
+- PR #6303 Use cudf test dtypes so timedelta tests are deterministic
+- PR #6329 Update and clean-up gpuCI scripts
 - PR #6299 Add lead and lag to java
-- PR #6303 Use cudf test dtypes so timedelta tests are determinstic
 - PR #6327 Add dictionary specialization to `cudf::replace_nulls`
 - PR #6306 Remove cpw macros from page encode kernels
 - PR #6375 Parallelize Cython compilation in addition to Cythonization
@@ -168,7 +169,7 @@
 - PR #6294 Fix read parquet key error when reading empty pandas DataFrame with cudf
 - PR #6285 Removed unsafe `reinterpret_cast` and implicit pointer-to-bool casts
 - PR #6281 Fix unreachable code warning in datetime.cuh
-- PR #6286 Fix `read_csv` `int32` overflow 
+- PR #6286 Fix `read_csv` `int32` overflow
 - PR #6466 Fix ORC reader issue with decimal type
 - PR #6310 Replace a misspelled reference to `master` branch with `main` branch in a comment in changelog.sh
 - PR #6289 Revert #6206

ci/benchmark/build.sh

@@ -58,7 +58,8 @@ logger "Check GPU usage..."
 nvidia-smi
 
 logger "Activate conda env..."
-source activate rapids
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
 # Enter dependencies to be shown in ASV tooltips.
 CUDF_DEPS=(librmm)
@@ -85,7 +86,9 @@ logger "Check versions..."
 python --version
 $CC --version
 $CXX --version
-conda list
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 ################################################################################
 # BUILD - Build libcudf, cuDF and dask_cudf from source
@@ -164,7 +167,7 @@ set -e
 rm ${WORKSPACE}/tmp/benchmark/benchmarks.txt
 cd ${WORKSPACE}
 mv ${GBENCH_BENCHMARKS_DIR}/*.json ${WORKSPACE}/tmp/benchmark/
-python GBenchToASV.py -d  ${WORKSPACE}/tmp/benchmark/ -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}" 
+python GBenchToASV.py -d  ${WORKSPACE}/tmp/benchmark/ -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}"
 
 ###
 # Run Python Benchmarks

ci/cpu/build.sh

@@ -1,22 +1,24 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
-######################################
-# cuDF CPU conda build script for CI #
-######################################
+# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+##############################################
+# cuDF CPU conda build script for CI         #
+##############################################
 set -e
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
 
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 # Switch to project root; also root of repo checkout
 cd $WORKSPACE
 
@@ -29,45 +31,51 @@ fi
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Check environment variables"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check compiler versions"
 python --version
-gcc --version
-g++ --version
-conda list
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
 ################################################################################
-# BUILD - Conda package builds (conda deps: libcudf <- libcudf_cffi <- cudf)
+# BUILD - Conda package builds
 ################################################################################
 
-logger "Build conda pkg for libcudf..."
-source ci/cpu/libcudf/build_libcudf.sh
+gpuci_logger "Build conda pkg for libcudf"
+gpuci_conda_retry build conda/recipes/libcudf
+
+gpuci_logger "Build conda pkg for libcudf_kafka"
+gpuci_conda_retry build conda/recipes/libcudf_kafka
 
-logger "Build conda pkg for libcudf_kafka..."
-source ci/cpu/libcudf_kafka/build_libcudf_kafka.sh
+gpuci_logger "Build conda pkg for cudf"
+gpuci_conda_retry build conda/recipes/cudf --python=$PYTHON
 
-logger "Build conda pkg for cudf..."
-source ci/cpu/cudf/build_cudf.sh
+gpuci_logger "Build conda pkg for dask-cudf"
+gpuci_conda_retry build conda/recipes/dask-cudf --python=$PYTHON
 
-logger "Build conda pkg for dask-cudf..."
-source ci/cpu/dask-cudf/build_dask_cudf.sh
+gpuci_logger "Build conda pkg for cudf_kafka"
+gpuci_conda_retry build conda/recipes/cudf_kafka --python=$PYTHON
 
-logger "Build conda pkg for cudf_kafka..."
-source ci/cpu/cudf_kafka/build_cudf_kafka.sh
+gpuci_logger "Build conda pkg for custreamz"
+gpuci_conda_retry build conda/recipes/custreamz --python=$PYTHON
 
-logger "Build conda pkg for custreamz..."
-source ci/cpu/custreamz/build_custreamz.sh
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
-logger "Upload conda pkgs..."
-source ci/cpu/upload_anaconda.sh
+gpuci_logger "Upload conda pkgs"
+source ci/cpu/upload.sh

ci/cpu/upload.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 3 total attempts)
+export GPUCI_RETRY_MAX=2
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+export LIBCUDF_FILE=`conda build conda/recipes/libcudf --output`
+export LIBCUDF_KAFKA_FILE=`conda build conda/recipes/libcudf_kafka --output`
+export CUDF_FILE=`conda build conda/recipes/cudf --python=$PYTHON --output`
+export DASK_CUDF_FILE=`conda build conda/recipes/dask-cudf --python=$PYTHON --output`
+export CUDF_KAFKA_FILE=`conda build conda/recipes/cudf_kafka --python=$PYTHON --output`
+export CUSTREAMZ_FILE=`conda build conda/recipes/custreamz --python=$PYTHON --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Staring conda uploads"
+if [ "$UPLOAD_LIBCUDF" == "1" ]; then
+  test -e ${LIBCUDF_FILE}
+  echo "Upload libcudf"
+  echo ${LIBCUDF_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_FILE}
+fi
+
+if [ "$UPLOAD_CUDF" == "1" ]; then
+  test -e ${CUDF_FILE}
+  echo "Upload cudf"
+  echo ${CUDF_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_FILE}
+
+  test -e ${DASK_CUDF_FILE}
+  echo "Upload dask-cudf"
+  echo ${DASK_CUDF_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK_CUDF_FILE}
+
+  test -e ${CUSTREAMZ_FILE}
+  echo "Upload custreamz"
+  echo ${CUSTREAMZ_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE}
+fi
+
+if [ "$UPLOAD_LIBCUDF_KAFKA" == "1" ]; then
+  test -e ${LIBCUDF_KAFKA_FILE}
+  echo "Upload libcudf_kafka"
+  echo ${LIBCUDF_KAFKA_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_KAFKA_FILE}
+fi
+
+if [ "$UPLOAD_CUDF_KAFKA" == "1" ]; then
+  test -e ${CUDF_KAFKA_FILE}
+  echo "Upload cudf_kafka"
+  echo ${CUDF_KAFKA_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_KAFKA_FILE}
+fi