Gpuciscripts clean and update

CHANGELOG.md

@@ -4,6 +4,7 @@
 - PR #208 Adds support for new dtype - datetime for all chart types except choropleths, Added new chart widget type - DateRangeSlider
 ## Improvements
 - PR #208 refactor - merged BaseLine and BaseBar to BaseAggregate
+- PR #215 cleand up gpuCI scripts
 ## Bug Fixes
 - PR #209 remove deprecated cudf methods- `to_gpu_matrix`, `add_column` and groupby parameter `method`
 

ci/checks/changelog.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# COPYRIGHT (c) 2020, NVIDIA CORPORATION.
 #########################
 # cuxfilter CHANGELOG Tester #
 #########################

ci/checks/style.sh

@@ -1,11 +1,12 @@
 # Ignore errors and set path
 set +e
-PATH=/conda/bin:$PATH
+PATH=/opt/conda/bin:$PATH
 LC_ALL=C.UTF-8
 LANG=C.UTF-8
 
 # Activate common conda env
-source activate gdf
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
 # Run black and get results/return code
 BLACK=`black --check python`

ci/cpu/build.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# COPYRIGHT (c) 2020, NVIDIA CORPORATION.
 ######################################
 # cuXfilter CPU conda build script for CI #
 ######################################
@@ -11,12 +11,19 @@ function logger() {
 }
 
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
 
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 # Switch to project root; also root of repo checkout
 cd $WORKSPACE
 
@@ -29,32 +36,37 @@ fi
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Check environment variables"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check compiler versions"
 python --version
-gcc --version
-g++ --version
-conda list
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
 ################################################################################
-# BUILD - Conda package build
+# BUILD - Conda package builds
 ################################################################################
 
-logger "Build conda pkg for cuxfilter..."
-source ci/cpu/cuxfilter/build_cuxfilter.sh
+echo "Building cuxfilter"
+gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON
 
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
-logger "Upload conda pkgs..."
-source ci/cpu/upload_anaconda.sh
+gpuci_logger "Upload conda pkgs"
+source ci/cpu/upload.sh
 

ci/cpu/upload.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+export CUXFILTER_FILE=`conda build conda/recipes/cuxfilter --python=$PYTHON --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+
+if [ "$UPLOAD_CUXFILTER" == "1" ]; then
+  LABEL_OPTION="--label main"
+  echo "LABEL_OPTION=${LABEL_OPTION}"
+
+  test -e ${CUXFILTER_FILE}
+  echo "Upload cuXfilter"
+  echo ${CUXFILTER_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUXFILTER_FILE}
+fi

ci/docs/build.sh

@@ -11,31 +11,34 @@ if [ -z "$PROJECT_WORKSPACE" ]; then
 fi
 
 export DOCS_WORKSPACE=$WORKSPACE/docs
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export HOME=$WORKSPACE
 export PROJECT_WORKSPACE=/rapids/cuxfilter
 export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 export NIGHTLY_VERSION=$(echo $BRANCH_VERSION | awk -F. '{print $2}')
 export PROJECTS=(cuxfilter)
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # Build Python docs
-logger "Build Sphinx docs..."
+gpuci_logger "Build Sphinx docs"
 cd $PROJECT_WORKSPACE/docs
 make html
 

ci/gpu/build.sh

@@ -1,25 +1,20 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# COPYRIGHT (c) 2020, NVIDIA CORPORATION.
 ##############################################
 # cuXfilter GPU build and test script for CI #
 ##############################################
 set -e
 NUMARGS=$#
 ARGS=$*
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Arg parsing function
 function hasArg {
     (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 export CUDA_REL=${CUDA_VERSION%.*}
 
 # Set home to the job's workspace
@@ -36,27 +31,28 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?
-    logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH"
-    rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH"
+    gpuci_logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH"
+    rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || gpuci_logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH"
     exit $EXITCODE
 }
 
 trap remove_libcudf_kernel_cache_dir EXIT
 
-mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH"
+mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || gpuci_logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH"
 
 ################################################################################
 # SETUP - Check environment
 ################################################################################
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 conda install "cudf=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
                "cugraph=$MINOR_VERSION.*" \
                "cuspatial=$MINOR_VERSION.*" \
@@ -69,17 +65,19 @@ conda install "cudf=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
 # conda remove -f rapids-build-env rapids-notebook-env
 # conda install "your-pkg=1.0.0"
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 ################################################################################
 # BUILD - Build cuxfilter from source
 ################################################################################
 
-logger "Build cuxfilter..."
+gpuci_logger "Build cuxfilter"
 $WORKSPACE/build.sh clean cuxfilter
 
 ################################################################################
@@ -91,13 +89,13 @@ EXITCODE=0
 trap "EXITCODE=1" ERR
 
 if hasArg --skip-tests; then
-    logger "Skipping Tests..."
+    gpuci_logger "Skipping Tests"
 else
-    logger "Check GPU usage..."
+    gpuci_logger "Check GPU usage"
     nvidia-smi
 
     cd $WORKSPACE/python/cuxfilter/tests
-    logger "Python py.test for cuxfilter..."
+    gpuci_logger "Python py.test for cuxfilter"
     py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuxfilter.xml -v
 
     ${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log

ci/local/README.md

@@ -30,7 +30,8 @@ For a full list of available gpuCI docker images, visit our [DockerHub](https://
 Style Check:
 ```bash
 $ bash ci/local/build.sh -r ~/rapids/cuxfilter -s
-$ source activate rapids    # Activate gpuCI conda environment
+$ . /opt/conda/etc/profile.d/conda.sh
+$ conda activate rapids   # Activate gpuCI conda environment
 $ cd rapids
 $ flake8 python
 ```