From de6a5076a5654df169b66069a047c3091c037efb Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 6 Nov 2020 12:49:17 -0800 Subject: [PATCH 1/6] Gpuciscripts clean and update --- ci/checks/changelog.sh | 2 +- ci/checks/style.sh | 5 ++-- ci/cpu/build.sh | 33 +++++++++++++++---------- ci/cpu/cuxfilter/build_cuxfilter.sh | 2 +- ci/cpu/upload_anaconda.sh | 2 +- ci/docs/build.sh | 19 ++++++++------- ci/gpu/build.sh | 37 ++++++++++++++++------------- ci/local/README.md | 3 ++- 8 files changed, 59 insertions(+), 44 deletions(-) diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh index 15229d87..298a00ca 100644 --- a/ci/checks/changelog.sh +++ b/ci/checks/changelog.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. +# COPYRIGHT (c) 2020, NVIDIA CORPORATION. ######################### # cuxfilter CHANGELOG Tester # ######################### diff --git a/ci/checks/style.sh b/ci/checks/style.sh index 3ff61a01..dff65164 100644 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -1,11 +1,12 @@ # Ignore errors and set path set +e -PATH=/conda/bin:$PATH +PATH=/opt/conda/bin:$PATH LC_ALL=C.UTF-8 LANG=C.UTF-8 # Activate common conda env -source activate gdf +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids # Run black and get results/return code BLACK=`black --check python` diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 77cfaa70..e30bef7c 100644 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -1,18 +1,18 @@ #!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. +# COPYRIGHT (c) 2020, NVIDIA CORPORATION. ###################################### # cuXfilter CPU conda build script for CI # ###################################### set -e # Logger function for build status output -function logger() { +function gpuci_logger() { echo -e "\n>>>> $@\n" } # Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=4 +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=-4 # Set home to the job's workspace export HOME=$WORKSPACE @@ -25,21 +25,28 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then export VERSION_SUFFIX=`date +%y%m%d` fi +# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) +export GPUCI_CONDA_RETRY_MAX=1 +export GPUCI_CONDA_RETRY_SLEEP=30 + ################################################################################ # SETUP - Check environment ################################################################################ -logger "Get env..." +gpuci_logger "Get env" env -logger "Activate conda env..." -source activate gdf +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids -logger "Check versions..." +gpuci_logger "Check versions" python --version -gcc --version -g++ --version -conda list +$CC --version +$CXX --version +conda info +conda config --show-sources +conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False @@ -48,13 +55,13 @@ conda config --set ssl_verify False # BUILD - Conda package build ################################################################################ -logger "Build conda pkg for cuxfilter..." +gpuci_logger "Build conda pkg for cuxfilter" source ci/cpu/cuxfilter/build_cuxfilter.sh ################################################################################ # UPLOAD - Conda packages ################################################################################ -logger "Upload conda pkgs..." +gpuci_logger "Upload conda pkgs" source ci/cpu/upload_anaconda.sh diff --git a/ci/cpu/cuxfilter/build_cuxfilter.sh b/ci/cpu/cuxfilter/build_cuxfilter.sh index de0d5f7b..dd34ad0f 100644 --- a/ci/cpu/cuxfilter/build_cuxfilter.sh +++ b/ci/cpu/cuxfilter/build_cuxfilter.sh @@ -1,4 +1,4 @@ set -e echo "Building cuxfilter" -conda build conda/recipes/cuxfilter --python=$PYTHON +gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON diff --git a/ci/cpu/upload_anaconda.sh b/ci/cpu/upload_anaconda.sh index af3911b3..6d8c26d6 100644 --- a/ci/cpu/upload_anaconda.sh +++ b/ci/cpu/upload_anaconda.sh @@ -4,7 +4,7 @@ set -e -export CUXFILTER_FILE=`conda build conda/recipes/cuxfilter --python=$PYTHON --output` +export CUXFILTER_FILE=`gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON --output` CUDA_REL=${CUDA_VERSION%.*} diff --git a/ci/docs/build.sh b/ci/docs/build.sh index 80fd9f0a..b6382e06 100644 --- a/ci/docs/build.sh +++ b/ci/docs/build.sh @@ -11,31 +11,34 @@ if [ -z "$PROJECT_WORKSPACE" ]; then fi export DOCS_WORKSPACE=$WORKSPACE/docs -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export HOME=$WORKSPACE export PROJECT_WORKSPACE=/rapids/cuxfilter export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" export NIGHTLY_VERSION=$(echo $BRANCH_VERSION | awk -F. '{print $2}') export PROJECTS=(cuxfilter) -logger "Check environment..." +gpuci_logger "Check environment" env -logger "Check GPU usage..." +gpuci_logger "Check GPU usage" nvidia-smi -logger "Activate conda env..." -source activate rapids +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids -logger "Check versions..." +gpuci_logger "Check versions" python --version $CC --version $CXX --version -conda list +conda info +conda config --show-sources +conda list --show-channel-urls # Build Python docs -logger "Build Sphinx docs..." +gpuci_logger "Build Sphinx docs" cd $PROJECT_WORKSPACE/docs make html diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index e9faf66d..c0fc25a2 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. +# COPYRIGHT (c) 2020, NVIDIA CORPORATION. ############################################## # cuXfilter GPU build and test script for CI # ############################################## @@ -8,7 +8,7 @@ NUMARGS=$# ARGS=$* # Logger function for build status output -function logger() { +function gpuci_logger() { echo -e "\n>>>> $@\n" } @@ -18,8 +18,8 @@ function hasArg { } # Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=4 +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=-4 export CUDA_REL=${CUDA_VERSION%.*} # Set home to the job's workspace @@ -36,27 +36,28 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" function remove_libcudf_kernel_cache_dir { EXITCODE=$? - logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" + gpuci_logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" + rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || gpuci_logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" exit $EXITCODE } trap remove_libcudf_kernel_cache_dir EXIT -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" +mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || gpuci_logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" ################################################################################ # SETUP - Check environment ################################################################################ -logger "Check environment..." +gpuci_logger "Check environment" env -logger "Check GPU usage..." +gpuci_logger "Check GPU usage" nvidia-smi -logger "Activate conda env..." -source activate rapids +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids conda install "cudf=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ "cugraph=$MINOR_VERSION.*" \ "cuspatial=$MINOR_VERSION.*" \ @@ -69,17 +70,19 @@ conda install "cudf=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ # conda remove -f rapids-build-env rapids-notebook-env # conda install "your-pkg=1.0.0" -logger "Check versions..." +gpuci_logger "Check versions" python --version $CC --version $CXX --version -conda list +conda info +conda config --show-sources +conda list --show-channel-urls ################################################################################ # BUILD - Build cuxfilter from source ################################################################################ -logger "Build cuxfilter..." +gpuci_logger "Build cuxfilter" $WORKSPACE/build.sh clean cuxfilter ################################################################################ @@ -91,13 +94,13 @@ EXITCODE=0 trap "EXITCODE=1" ERR if hasArg --skip-tests; then - logger "Skipping Tests..." + gpuci_logger "Skipping Tests" else - logger "Check GPU usage..." + gpuci_logger "Check GPU usage" nvidia-smi cd $WORKSPACE/python/cuxfilter/tests - logger "Python py.test for cuxfilter..." + gpuci_logger "Python py.test for cuxfilter" py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuxfilter.xml -v ${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log diff --git a/ci/local/README.md b/ci/local/README.md index a784a472..c7a97cdf 100644 --- a/ci/local/README.md +++ b/ci/local/README.md @@ -30,7 +30,8 @@ For a full list of available gpuCI docker images, visit our [DockerHub](https:// Style Check: ```bash $ bash ci/local/build.sh -r ~/rapids/cuxfilter -s -$ source activate rapids # Activate gpuCI conda environment +$ . /opt/conda/etc/profile.d/conda.sh +$ conda activate rapids # Activate gpuCI conda environment $ cd rapids $ flake8 python ``` From f1643f35bb8e16db49836e0f5a3e2a4b0c24e227 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Thu, 12 Nov 2020 20:41:04 -0800 Subject: [PATCH 2/6] Updated upload.sh and build.sh --- ci/cpu/build.sh | 32 ++++++++++---------- ci/cpu/cuxfilter/build_cuxfilter.sh | 4 --- ci/cpu/upload.sh | 47 +++++++++++++++++++++++++++++ ci/cpu/upload_anaconda.sh | 29 ------------------ 4 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 ci/cpu/cuxfilter/build_cuxfilter.sh create mode 100644 ci/cpu/upload.sh delete mode 100644 ci/cpu/upload_anaconda.sh diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index e30bef7c..ada1b68d 100644 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -5,18 +5,20 @@ ###################################### set -e -# Logger function for build status output -function gpuci_logger() { - echo -e "\n>>>> $@\n" -} - # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=-4 +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} # Set home to the job's workspace export HOME=$WORKSPACE +# Determine CUDA release version +export CUDA_REL=${CUDA_VERSION%.*} + +# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) +export GPUCI_CONDA_RETRY_MAX=1 +export GPUCI_CONDA_RETRY_SLEEP=30 + # Switch to project root; also root of repo checkout cd $WORKSPACE @@ -25,25 +27,23 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then export VERSION_SUFFIX=`date +%y%m%d` fi -# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) -export GPUCI_CONDA_RETRY_MAX=1 -export GPUCI_CONDA_RETRY_SLEEP=30 - ################################################################################ # SETUP - Check environment ################################################################################ -gpuci_logger "Get env" +gpuci_logger "Check environment variables" env gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -gpuci_logger "Check versions" +gpuci_logger "Check compiler versions" python --version $CC --version $CXX --version + +gpuci_logger "Check conda environment" conda info conda config --show-sources conda list --show-channel-urls @@ -52,16 +52,16 @@ conda list --show-channel-urls conda config --set ssl_verify False ################################################################################ -# BUILD - Conda package build +# BUILD - Conda package builds ################################################################################ -gpuci_logger "Build conda pkg for cuxfilter" -source ci/cpu/cuxfilter/build_cuxfilter.sh +echo "Building cuxfilter" +gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON ################################################################################ # UPLOAD - Conda packages ################################################################################ gpuci_logger "Upload conda pkgs" -source ci/cpu/upload_anaconda.sh +source ci/cpu/upload.sh diff --git a/ci/cpu/cuxfilter/build_cuxfilter.sh b/ci/cpu/cuxfilter/build_cuxfilter.sh deleted file mode 100644 index dd34ad0f..00000000 --- a/ci/cpu/cuxfilter/build_cuxfilter.sh +++ /dev/null @@ -1,4 +0,0 @@ -set -e - -echo "Building cuxfilter" -gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh new file mode 100644 index 00000000..71aa2a39 --- /dev/null +++ b/ci/cpu/upload.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh + +set -e + +# Setup 'gpuci_retry' for upload retries (results in 4 total attempts) +export GPUCI_RETRY_MAX=3 +export GPUCI_RETRY_SLEEP=30 + +# Set default label options if they are not defined elsewhere +export LABEL_OPTION=${LABEL_OPTION:-"--label main"} + +# Skip uploads unless BUILD_MODE == "branch" +if [ ${BUILD_MODE} != "branch" ]; then + echo "Skipping upload" + return 0 +fi + +# Skip uploads if there is no upload key +if [ -z "$MY_UPLOAD_KEY" ]; then + echo "No upload key" + return 0 +fi + +################################################################################ +# SETUP - Get conda file output locations +################################################################################ + +gpuci_logger "Get conda file output locations" +export CUXFILTER_FILE=`gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON --output` + +################################################################################ +# UPLOAD - Conda packages +################################################################################ + +gpuci_logger "Starting conda uploads" + +if [ "$UPLOAD_CUXFILTER" == "1" ]; then + LABEL_OPTION="--label main" + echo "LABEL_OPTION=${LABEL_OPTION}" + + test -e ${CUXFILTER_FILE} + echo "Upload cuXfilter" + echo ${CUXFILTER_FILE} + anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUXFILTER_FILE} +fi diff --git a/ci/cpu/upload_anaconda.sh b/ci/cpu/upload_anaconda.sh deleted file mode 100644 index 6d8c26d6..00000000 --- a/ci/cpu/upload_anaconda.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh - -set -e - -export CUXFILTER_FILE=`gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON --output` - -CUDA_REL=${CUDA_VERSION%.*} - -if [ ${BUILD_MODE} != "branch" ]; then - echo "Skipping upload" - return 0 -fi - -if [ -z "$MY_UPLOAD_KEY" ]; then - echo "No upload key" - return 0 -fi - -if [ "$UPLOAD_CUXFILTER" == "1" ]; then - LABEL_OPTION="--label main" - echo "LABEL_OPTION=${LABEL_OPTION}" - - test -e ${CUXFILTER_FILE} - echo "Upload cuXfilter" - echo ${CUXFILTER_FILE} - anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUXFILTER_FILE} -fi From fe2e6b39bdbab369e2372de8e2e744d2fdf738a2 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 07:59:12 -0800 Subject: [PATCH 3/6] added gpuci_logger --- ci/cpu/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index ada1b68d..b149718b 100644 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -5,6 +5,11 @@ ###################################### set -e +# Logger function for build status output +function logger() { + echo -e "\n>>>> $@\n" +} + # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} From cfaa8dba3970bb7177a22c616690a277ae0ecba4 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 08:25:37 -0800 Subject: [PATCH 4/6] added gpuci_retry --- ci/cpu/upload.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 71aa2a39..11de0f3a 100644 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -28,7 +28,7 @@ fi ################################################################################ gpuci_logger "Get conda file output locations" -export CUXFILTER_FILE=`gpuci_conda_retry build conda/recipes/cuxfilter --python=$PYTHON --output` +export CUXFILTER_FILE=`conda build conda/recipes/cuxfilter --python=$PYTHON --output` ################################################################################ # UPLOAD - Conda packages @@ -43,5 +43,5 @@ if [ "$UPLOAD_CUXFILTER" == "1" ]; then test -e ${CUXFILTER_FILE} echo "Upload cuXfilter" echo ${CUXFILTER_FILE} - anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUXFILTER_FILE} + gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUXFILTER_FILE} fi From 52e91b0f302982f193990c8c328ed703176dfced Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 10:28:13 -0800 Subject: [PATCH 5/6] removed gpuci_logger and replaced parallel_level --- ci/gpu/build.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index c0fc25a2..bf1142d9 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -7,11 +7,6 @@ set -e NUMARGS=$# ARGS=$* -# Logger function for build status output -function gpuci_logger() { - echo -e "\n>>>> $@\n" -} - # Arg parsing function function hasArg { (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") @@ -19,7 +14,7 @@ function hasArg { # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=-4 +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} export CUDA_REL=${CUDA_VERSION%.*} # Set home to the job's workspace From 2c57c5afcef6cc851fdfad8d350400f3577b7164 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Wed, 2 Dec 2020 14:52:54 -0800 Subject: [PATCH 6/6] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a563c90f..41cecd72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - PR #208 Adds support for new dtype - datetime for all chart types except choropleths, Added new chart widget type - DateRangeSlider ## Improvements - PR #208 refactor - merged BaseLine and BaseBar to BaseAggregate +- PR #215 cleand up gpuCI scripts ## Bug Fixes - PR #209 remove deprecated cudf methods- `to_gpu_matrix`, `add_column` and groupby parameter `method`