[REVIEW] ENH Improve gpuCI scripts

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ## Bug Fixes
 - PR #272 Fix bug in gausspulse
+- PR #275 Improve gpuCI Scripts
 
 # cuSignal 0.16.0 (Date TBD)
 

ci/cpu/build.sh

@@ -5,21 +5,20 @@
 ##########################################
 set -e
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
 
 # Switch to project root; also root of repo checkout
 cd $WORKSPACE
 
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 # If nightly build, append current YYMMDD to version
 if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
@@ -29,17 +28,22 @@ fi
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Get env"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 gcc --version
 g++ --version
-conda list
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
@@ -48,12 +52,12 @@ conda config --set ssl_verify False
 # BUILD - Conda package build
 ################################################################################
 
-logger "Build conda pkg for cuSignal..."
-conda build conda/recipes/cusignal --python=${PYTHON}
+gpuci_logger "Build conda pkg for cuSignal"
+gpuci_conda_retry build conda/recipes/cusignal --python=${PYTHON}
 
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
-logger "Upload conda pkgs for cuSignal..."
-source ci/cpu/upload-anaconda.sh
+gpuci_logger "Upload conda pkgs for cuSignal"
+source ci/cpu/upload.sh

ci/cpu/upload.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+
+export CUSIGNAL_FILE=`conda build conda/recipes/libcugraph --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+
+test -e ${CUSIGNAL_FILE}
+echo "Upload cusignal"
+echo ${CUSIGNAL_FILE}
+gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSIGNAL_FILE}
+

ci/docs/build.sh

@@ -18,27 +18,31 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 export NIGHTLY_VERSION=$(echo $BRANCH_VERSION | awk -F. '{print $2}')
 export PROJECTS=(cusignal)
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
-# TODO: Move installs to docs-build-env meta package
-conda install -c anaconda markdown beautifulsoup4 jq
-pip install sphinx-markdown-tables
+gpuci_logger "Activate conda env"
 
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # Build Python docs
-logger "Build Sphinx docs..."
+gpuci_logger "Build Sphinx docs"
 cd $PROJECT_WORKSPACE/docs
 make html
 
@@ -53,4 +57,3 @@ for PROJECT in ${PROJECTS[@]}; do
 done
 
 mv $PROJECT_WORKSPACE/docs/build/html/* $DOCS_WORKSPACE/api/cusignal/$BRANCH_VERSION
-

ci/gpu/build.sh

@@ -7,19 +7,14 @@ set -e
 NUMARGS=$#
 ARGS=$*
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Arg parsing function
 function hasArg {
     (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 export CUDA_REL=${CUDA_VERSION%.*}
 
 # Set home to the job's workspace
@@ -34,34 +29,40 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 # SETUP - Check environment
 ################################################################################
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
-conda install -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge \
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+gpuci_conda_retry install -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge \
     cudatoolkit=${CUDA_REL} \
     "rapids-build-env=$MINOR_VERSION.*" \
     "rapids-notebook-env=$MINOR_VERSION."
 
 # https://docs.rapids.ai/maintainers/depmgmt/ 
-# conda remove -f rapids-build-env rapids-notebook-env
-# conda install "your-pkg=1.0.0"
+# gpuci_conda_retry remove -f rapids-build-env rapids-notebook-env
+# gpuci_conda_retry install "your-pkg=1.0.0"
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 ################################################################################
 # BUILD - Build cusignal
 ################################################################################
 
-logger "Build cusignal..."
+gpuci_logger "Build cusignal"
 $WORKSPACE/build.sh clean cusignal
 
 ################################################################################
@@ -73,20 +74,20 @@ EXITCODE=0
 trap "EXITCODE=1" ERR
 
 if hasArg --skip-tests; then
-    logger "Skipping Tests..."
+    gpuci_logger "Skipping Tests"
     exit 0
 fi
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Python pytest for cusignal..."
+gpuci_logger "Python pytest for cusignal"
 cd $WORKSPACE/python
 
 pytest --cache-clear --junitxml=${WORKSPACE}/junit-cusignal.xml -v -s -m "not cpu"
 
 conda remove -y --force blas nomkl rapids-build-env rapids-notebook-env
-conda install -y -c pytorch "pytorch>=1.4"
+gpuci_conda_retry install -y -c pytorch "pytorch>=1.4"
 
 ${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log
 python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log