[RELEASE] integration v21.06

ci/axis/nightly.yaml

@@ -6,26 +6,15 @@ CONDA_CONFIG_FILE:
 
 # Use M.X.Ya (major.minor.patch) with 'a' version to match nightly tag
 RAPIDS_VER:
-  - 0.19.0a
-  - 0.20.0a
+  - 21.06.00a
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
 CUDA_VER:
   - 11.2
   - 11.0
-  - 10.2
-  - 10.1
-
-# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated
-DEFAULT_CUDA_VER:
-  - 10.1
 
 PYTHON_VER:
   - 3.7
   - 3.8
 
 exclude:
-  - RAPIDS_VER: 0.20.0a
-    CUDA_VER: 10.1
-  - RAPIDS_VER: 0.20.0a
-    CUDA_VER: 10.2

ci/axis/release.yaml

@@ -6,19 +6,15 @@ CONDA_CONFIG_FILE:
 
 # Use M.X.Y (major.minor.patch) version to match tag
 RAPIDS_VER:
-  - 0.19.0
+  - "21.06.00"
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
 CUDA_VER:
   - 11.2
   - 11.0
-  - 10.2
-  - 10.1
-
-# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated
-DEFAULT_CUDA_VER:
-  - 10.1
 
 PYTHON_VER:
   - 3.7
   - 3.8
+
+exclude:

ci/axis/tests.yaml

@@ -6,17 +6,20 @@ DOCKER_REPO:
 
 # Use M.X (major.minor) version
 RAPIDS_VER:
-  - 0.19
+  - '21.06'
 
 CUDA_VER:
+  - 11.2
   - 11.0
-  - 10.2
-  - 10.1
 
 LINUX_VER:
   - ubuntu16.04
   - ubuntu18.04
+  - ubuntu20.04
   - centos7
+  - centos8
 
 PYTHON_VER:
   - 3.7
+
+exclude:

ci/cpu/build.sh

@@ -14,7 +14,7 @@ set -e
 
 # Set paths
 export PATH=/conda/bin:$PATH
-export HOME=$WORKSPACE
+export HOME="$WORKSPACE"
 
 # Set recipe paths
 CONDA_XGBOOST_RECIPE="conda/recipes/rapids-xgboost"
@@ -53,13 +53,13 @@ function build_pkg {
   # Build pkg
   gpuci_logger "Start conda build for '${1}'..."
   if [[ "${1}" == "${CONDA_BLAZING_NOTEBOOK_RECIPE}" ]]; then
-    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c pytorch -c conda-forge -c defaults \
+    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c pytorch -c conda-forge \
                 --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
   elif [[ "${1}" == *"BLAZING"* ]]; then
-    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge -c defaults \
+    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge \
                 --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
   else
-    gpuci_conda_retry build --override-channels -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge -c defaults \
+    gpuci_conda_retry build --override-channels -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge \
                 --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
   fi
 }
@@ -70,27 +70,39 @@ function run_builds {
 }
 
 function upload_builds {
+
+  # Check arch
+  ARCH=$(uname -m)
+  if [ "${ARCH}" = "x86_64" ]; then
+    ARCH_DIR="linux-64"
+  elif [ "${ARCH}" = "aarch64" ]; then
+    ARCH_DIR="linux-aarch64"
+  else
+    echo "ERROR: Unsupported arch: ${ARCH}"
+    exit 1
+  fi
+
   # Check for upload key
   if [ -z "$MY_UPLOAD_KEY" ]; then
     gpuci_logger "No upload key found, env var MY_UPLOAD_KEY not set, skipping upload..."
   else
     gpuci_logger "Upload key found, starting upload..."
     gpuci_logger "Files to upload..."
-    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2) ]]; then
-      ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2
+    if [[ -n $(ls /conda/conda-bld/${ARCH_DIR}/* | grep -i rapids.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/${ARCH_DIR}/* | grep -i rapids.*.tar.bz2
     fi
-    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2) ]]; then
-      ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2
+    if [[ -n $(ls /conda/conda-bld/${ARCH_DIR}/* | grep -i blazingsql.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/${ARCH_DIR}/* | grep -i blazingsql.*.tar.bz2
     fi
 
     gpuci_logger "Starting upload..."
-    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2) ]]; then
-      ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2 | xargs gpuci_retry \
-        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing
+    if [[ -n $(ls /conda/conda-bld/${ARCH_DIR}/* | grep -i rapids.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/${ARCH_DIR}/* | grep -i rapids.*.tar.bz2 | xargs gpuci_retry \
+        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing --no-progress
     fi
-    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2) ]]; then
-      ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2 | xargs gpuci_retry \
-        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing
+    if [[ -n $(ls /conda/conda-bld/${ARCH_DIR}/* | grep -i blazingsql.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/${ARCH_DIR}/* | grep -i blazingsql.*.tar.bz2 | xargs gpuci_retry \
+        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing --no-progress
     fi
   fi
 }
@@ -99,9 +111,7 @@ if [[ "$BUILD_PKGS" == "meta" || -z "$BUILD_PKGS" ]] ; then
   # Run builds for meta-pkgs
   run_builds $CONDA_XGBOOST_RECIPE
   run_builds $CONDA_RAPIDS_RECIPE
-  set +e
   run_builds $CONDA_RAPIDS_BLAZING_RECIPE
-  set -e
 fi
 
 if [[ "$BUILD_PKGS" == "env" || -z "$BUILD_PKGS" ]] ; then

ci/cpu/conda-pack.sh

@@ -7,7 +7,7 @@ conda create -y --name=$CONDA_ENV_NAME python=$PYTHON_VER
 source activate $CONDA_ENV_NAME
 
 echo "Installing conda packages"
-conda install -y -c $CONDA_USERNAME -c nvidia -c conda-forge -c defaults \
+conda install -y -c $CONDA_USERNAME -c nvidia -c conda-forge \
     rapids=$RAPIDS_VER \
     cudatoolkit=$CUDA_VER \
     conda-pack \

ci/test/cudf.sh

@@ -2,8 +2,8 @@
 set +e
 set -x
 
-export HOME=$WORKSPACE
-export LIBCUDF_KERNEL_CACHE_PATH=$WORKSPACE/.cache/rapids/cudf
+export HOME="$WORKSPACE"
+export LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.cache/rapids/cudf"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -12,7 +12,7 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 

ci/test/cugraph.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 set +e
 set -x
-export HOME=$WORKSPACE
-export LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache
+export HOME="${WORKSPACE}"
+export LIBCUDF_KERNEL_CACHE_PATH="${WORKSPACE}/.jitcache"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -18,12 +18,15 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+# Install pytest plugin for cugraph
+conda install rapids-pytest-benchmark
+
+TESTRESULTS_DIR="${WORKSPACE}/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 
 # gtests
-for gt in /rapids/cugraph/cpp/build/gtests/*_TEST; do
+for gt in /rapids/cugraph/cpp/build/tests/*_TEST; do
    # FIXME: remove this ASAP
    ${gt} --gtest_output=xml:${TESTRESULTS_DIR}/
    exitcode=$?
@@ -34,7 +37,7 @@ for gt in /rapids/cugraph/cpp/build/gtests/*_TEST; do
 done
 
 # Python tests
-py.test --junitxml=${TESTRESULTS_DIR}/pytest.xml -v /rapids/cugraph/python
+pytest --ignore=/rapids/cugraph/python/cugraph/raft --junitxml=${TESTRESULTS_DIR}/pytest.xml -v /rapids/cugraph/python
 exitcode=$?
 if (( ${exitcode} != 0 )); then
    SUITEERROR=${exitcode}

ci/test/cuml.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 set +e
 
-export HOME=$WORKSPACE
-export LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache
+export HOME="$WORKSPACE"
+export LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -11,11 +11,11 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 
-export CUPY_CACHE_DIR=${WORKSPACE}/tmp
+export CUPY_CACHE_DIR="$WORKSPACE/tmp"
 mkdir -p ${CUPY_CACHE_DIR}
 
 # gtests

ci/test/cuspatial.sh

@@ -2,7 +2,7 @@
 set -ex
 
 export CUSPATIAL_HOME=/rapids/cuspatial
-export HOME=$WORKSPACE
+export HOME="$WORKSPACE"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -11,7 +11,7 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 

ci/test/daskcuda.sh

@@ -2,7 +2,7 @@
 set +e
 set -x
 
-export HOME=$WORKSPACE 
+export HOME="$WORKSPACE" 
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -12,7 +12,7 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 

ci/test/integration.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
-export HOME=${WORKSPACE}
-export LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache
+export HOME="$WORKSPACE"
+export LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 . /opt/conda/etc/profile.d/conda.sh
@@ -12,15 +12,15 @@ env
 nvidia-smi
 conda list
 
-export TESTRESULTS_DIR=${WORKSPACE}/testresults
+export TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 
 gpuci_logger "Install conda packages needed by tests in rapids environment"
 gpuci_conda_retry --condaretry_max_retries=10 install -y --freeze-installed requests
 
 gpuci_logger "Run Python tests"
-py.test --junitxml=${TESTRESULTS_DIR}/pytest.xml -v $WORKSPACE/test
+py.test --junitxml=${TESTRESULTS_DIR}/pytest.xml -v "$WORKSPACE/test"
 exitcode=$?
 if (( ${exitcode} != 0 )); then
    SUITEERROR=${exitcode}

ci/test/notebooks.sh

@@ -3,7 +3,7 @@ set +e
 set -x
 set -o pipefail
 
-export LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache
+export LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache"
 
 source /opt/conda/bin/activate rapids
 

ci/test/rmm.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -ex
-export HOME=$WORKSPACE
+export HOME="$WORKSPACE"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 # FIXME: "source activate" line should not be needed
@@ -9,7 +9,7 @@ env
 nvidia-smi
 conda list
 
-TESTRESULTS_DIR=${WORKSPACE}/testresults
+TESTRESULTS_DIR="$WORKSPACE/testresults"
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 

conda/recipes/blazingsql-build-env/meta.yaml

@@ -1,5 +1,5 @@
 {% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
-{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VER', '10.0').split('.')[:2]) %}
 {% set py_version = environ.get('CONDA_PY', 36) %}
 
 ### 

conda/recipes/blazingsql-notebook-env/meta.yaml

@@ -1,5 +1,5 @@
 {% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
-{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VER', '10.0').split('.')[:2]) %}
 {% set py_version = environ.get('CONDA_PY', 36) %}
 
 ### 

conda/recipes/rapids-blazing/meta.yaml

@@ -1,6 +1,6 @@
 {% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
 {% set minor_version = rapids_version.split('.')[0] + '.' + rapids_version.split('.')[1] %}
-{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VER', '10.0').split('.')[:2]) %}
 {% set py_version = environ.get('CONDA_PY', 36) %}
 
 ###

conda/recipes/rapids-build-env/meta.yaml

@@ -2,7 +2,7 @@
 
 {% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
 {% set minor_version = rapids_version.split('.')[0] + '.' + rapids_version.split('.')[1] %}
-{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VER', '10.0').split('.')[:2]) %}
 {% set py_version = environ.get('CONDA_PY', 36) %}
 
 ###
@@ -78,6 +78,7 @@ requirements:
     - gmock {{ gmock_version }}
     - graphviz
     - gtest {{ gtest_version }}
+    - hdbscan
     - httpretty
     - hypothesis
     - isort {{ isort_version }}
@@ -103,6 +104,7 @@ requirements:
     - numba {{ numba_version }}
     - numpy {{ numpy_version }}
     - nvtx {{ nvtx_version }}
+    - openslide
     - packaging
     - pandas {{ pandas_version }}
     - panel {{ panel_version }}
@@ -112,6 +114,7 @@ requirements:
     - psutil
     - pyarrow {{ arrow_version }}
     - pydeck {{ pydeck_version }}
+    - pydocstyle {{ pydocstyle_version }}
     - pynvml
     - pyorc
     - pyppeteer {{ pyppeteer_version }}