[RELEASE] v0.17 integration release

ci/axis/nightly.yaml

@@ -4,8 +4,8 @@ CONDA_USERNAME:
 CONDA_CONFIG_FILE:
   - conda/recipes/versions.yaml
 
+# Use M.X.Ya (major.minor.patch) with 'a' version to match nightly tag
 RAPIDS_VER:
-  - 0.16.0a
   - 0.17.0a
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
@@ -14,6 +14,7 @@ CUDA_VER:
   - 10.2
   - 10.1
 
+# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated
 DEFAULT_CUDA_VER:
   - 10.1
 

ci/axis/release.yaml

@@ -4,15 +4,17 @@ CONDA_USERNAME:
 CONDA_CONFIG_FILE:
   - conda/recipes/versions.yaml
 
+# Use M.X.Y (major.minor.patch) version to match tag
 RAPIDS_VER:
-  - 0.16
+  - 0.17.0
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
 CUDA_VER:
   - 11.0
   - 10.2
   - 10.1
 
+# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated
 DEFAULT_CUDA_VER:
   - 10.1
 

ci/axis/tests.yaml

@@ -4,8 +4,9 @@ RUNTIME_DOCKER_REPO:
 DOCKER_REPO:
   - rapidsai/rapidsai-dev-nightly
 
+# Use M.X (major.minor) version
 RAPIDS_VER:
-  - 0.16
+  - 0.17
 
 CUDA_VER:
   - 11.0

ci/cpu/build.sh

@@ -16,15 +16,15 @@ set -e
 export PATH=/conda/bin:$PATH
 export HOME=$WORKSPACE
 
-# Save original build offset
-export ORIG_OFFSET=$RAPIDS_OFFSET
-
 # Set recipe paths
 CONDA_XGBOOST_RECIPE="conda/recipes/rapids-xgboost"
 CONDA_RAPIDS_RECIPE="conda/recipes/rapids"
+CONDA_RAPIDS_BLAZING_RECIPE="conda/recipes/rapids-blazing"
 CONDA_RAPIDS_BUILD_RECIPE="conda/recipes/rapids-build-env"
 CONDA_RAPIDS_NOTEBOOK_RECIPE="conda/recipes/rapids-notebook-env"
 CONDA_RAPIDS_DOC_RECIPE="conda/recipes/rapids-doc-env"
+CONDA_BLAZING_BUILD_RECIPE="conda/recipes/blazingsql-build-env"
+CONDA_BLAZING_NOTEBOOK_RECIPE="conda/recipes/blazingsql-notebook-env"
 
 # Allow insecure connections for conda-mirror
 echo "ssl_verify: False" >> /conda/.condarc
@@ -50,40 +50,23 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
 fi
 
 function build_pkg {
-  # Create 'conda_build_config.yaml' with CUDA version
-  gpuci_logger "Creating 'conda_build_config.yaml' with CUDA_VER='$CUDA_VER' to select correct pkgs..."
-  cat > ${1}/conda_build_config.yaml <<EOF
-cuda_compiler_version:
-- '$CUDA_VER'
-EOF
-  cat ${1}/conda_build_config.yaml
   # Build pkg
   gpuci_logger "Start conda build for '${1}'..."
-  gpuci_conda_retry build --override-channels -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge -c defaults \
-              --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
-}
-
-function build_default_pkg {
-  # Build default version if current version matches DEFAULT_CUDA_VER
-  if [ "$CUDA_VER" == "$DEFAULT_CUDA_VER" ] ; then
-    gpuci_logger "Current CUDA_VER '$CUDA_VER' is the DEFAULT_CUDA_VER, building package again with incremented build number..."
-    gpuci_logger "Previous build number '$RAPIDS_OFFSET'"
-    export RAPIDS_OFFSET=$((RAPIDS_OFFSET+1))
-    gpuci_logger "New build number '$RAPIDS_OFFSET'"
-    build_pkg $1
-    # Reset offset
-    export RAPIDS_OFFSET=$ORIG_OFFSET
-    gpuci_logger "Reset build number after default build '$RAPIDS_OFFSET'"
+  if [[ "${1}" == "${CONDA_BLAZING_NOTEBOOK_RECIPE}" ]]; then
+    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c pytorch -c conda-forge -c defaults \
+                --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
+  elif [[ "${1}" == *"BLAZING"* ]]; then
+    gpuci_conda_retry build --override-channels -c blazingsql-nightly -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge -c defaults \
+                --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
   else
-    gpuci_logger "Current CUDA_VER '$CUDA_VER' is not DEFAULT_CUDA_VER, skipping default build..."
+    gpuci_conda_retry build --override-channels -c ${CONDA_USERNAME:-rapidsai-nightly} -c nvidia -c conda-forge -c defaults \
+                --python=${PYTHON_VER} -m ${CONDA_CONFIG_FILE} ${1}
   fi
 }
 
 function run_builds {
-  # Kick off main pkg b
+  # Kick off main pkg build
   build_pkg $1
-  # Check and build default pkgs
-  build_default_pkg $1
 }
 
 function upload_builds {
@@ -93,25 +76,41 @@ function upload_builds {
   else
     gpuci_logger "Upload key found, starting upload..."
     gpuci_logger "Files to upload..."
-    ls /conda/conda-bld/linux-64/rapids*.tar.bz2
+    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2
+    fi
+    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2
+    fi
 
     gpuci_logger "Starting upload..."
-    ls /conda/conda-bld/linux-64/rapids*.tar.bz2 | xargs gpuci_retry \
-      anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing
+    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/linux-64/* | grep -i rapids.*.tar.bz2 | xargs gpuci_retry \
+        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing
+    fi
+    if [[ -n $(ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2) ]]; then
+      ls /conda/conda-bld/linux-64/* | grep -i blazingsql.*.tar.bz2 | xargs gpuci_retry \
+        anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai-nightly} --label main --skip-existing
+    fi
   fi
 }
 
 if [[ "$BUILD_PKGS" == "meta" || -z "$BUILD_PKGS" ]] ; then
   # Run builds for meta-pkgs
   run_builds $CONDA_XGBOOST_RECIPE
   run_builds $CONDA_RAPIDS_RECIPE
+  set +e
+  run_builds $CONDA_RAPIDS_BLAZING_RECIPE
+  set -e
 fi
 
 if [[ "$BUILD_PKGS" == "env" || -z "$BUILD_PKGS" ]] ; then
   # Run builds for env-pkgs
   run_builds $CONDA_RAPIDS_BUILD_RECIPE
   run_builds $CONDA_RAPIDS_NOTEBOOK_RECIPE
   run_builds $CONDA_RAPIDS_DOC_RECIPE
+  run_builds $CONDA_BLAZING_BUILD_RECIPE
+  run_builds $CONDA_BLAZING_NOTEBOOK_RECIPE
 fi
 
 # Upload builds

ci/cpu/conda-pack.sh

@@ -14,8 +14,8 @@ conda install -y -c $CONDA_USERNAME -c nvidia -c conda-forge -c defaults \
     ipykernel
 
 echo "Packing conda environment"
-conda-pack --quiet -n $CONDA_ENV_NAME -o ${CONDA_ENV_NAME}.tar.gz
+conda-pack --quiet --ignore-missing-files -n $CONDA_ENV_NAME -o ${CONDA_ENV_NAME}.tar.gz
 
 export AWS_DEFAULT_REGION="us-east-2"
 echo "Upload packed conda"
-aws s3 cp --quiet ${CONDA_ENV_NAME}.tar.gz s3://rapidsai-data/conda-pack/${CONDA_USERNAME}/${CONDA_ENV_NAME}.tar.gz
+aws s3 cp --quiet --acl public-read ${CONDA_ENV_NAME}.tar.gz s3://rapidsai-data/conda-pack/${CONDA_USERNAME}/${CONDA_ENV_NAME}.tar.gz

ci/test/cudf.sh

@@ -9,6 +9,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local
 # FIXME: "source activate" line should not be needed
 source /opt/conda/bin/activate rapids
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults

ci/test/cugraph.sh

@@ -15,25 +15,23 @@ export RAPIDS_DATASET_ROOT_DIR=/rapids/cugraph/datasets
 
 # Show environment
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults
 mkdir -p ${TESTRESULTS_DIR}
 SUITEERROR=0
 
-# FIXME: temporarily disabling all C++ tests for 0.16 due to intermittent
-# failures from what appears to be an issue with Thrust (which does not appear
-# to affect the Python API or notebooks). Re-enable once this issue is resolved
-# in 0.17.
 # gtests
-# for gt in /rapids/cugraph/cpp/build/gtests/*_TEST; do
-#    ${gt} --gtest_output=xml:${TESTRESULTS_DIR}/
-#    exitcode=$?
-#    if (( ${exitcode} != 0 )); then
-#       SUITEERROR=${exitcode}
-#       echo "FAILED: ${gt}"
-#    fi
-# done
+for gt in /rapids/cugraph/cpp/build/gtests/*_TEST; do
+   # FIXME: remove this ASAP
+   ${gt} --gtest_output=xml:${TESTRESULTS_DIR}/
+   exitcode=$?
+   if (( ${exitcode} != 0 )); then
+      SUITEERROR=${exitcode}
+      echo "FAILED: ${gt}"
+   fi
+done
 
 # Python tests
 py.test --junitxml=${TESTRESULTS_DIR}/pytest.xml -v /rapids/cugraph/python

ci/test/cuml.sh

@@ -8,6 +8,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local
 # FIXME: "source activate" line should not be needed
 source /opt/conda/bin/activate rapids
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults

ci/test/cuspatial.sh

@@ -8,6 +8,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local
 # FIXME: "source activate" line should not be needed
 source /opt/conda/bin/activate rapids
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults

ci/test/daskcuda.sh

@@ -9,6 +9,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local
 source /opt/conda/bin/activate rapids
 
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults

ci/test/integration.sh

@@ -9,6 +9,7 @@ conda activate rapids
 
 gpuci_logger "Show env and current conda list"
 env
+nvidia-smi
 conda list
 
 export TESTRESULTS_DIR=${WORKSPACE}/testresults

ci/test/notebooks.sh

@@ -10,7 +10,7 @@ source /opt/conda/bin/activate rapids
 # PyTorch is intentionally excluded from our Docker images due
 # to its size, but some notebooks still depend on it.
 case "${CUDA_VER}" in
-"10.1" | "10.2")
+"10.1" | "10.2" | "11.0")
     conda install -y -c pytorch "pytorch>=1.4"
     ;;
 *)
@@ -21,6 +21,9 @@ esac
 
 
 env
+nvidia-smi
+conda list
+
 /test.sh 2>&1 | tee nbtest.log
 EXITCODE=$?
 python /rapids/utils/nbtestlog2junitxml.py nbtest.log

ci/test/rmm.sh

@@ -6,6 +6,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local
 # FIXME: "source activate" line should not be needed
 source /opt/conda/bin/activate rapids
 env
+nvidia-smi
 conda list
 
 TESTRESULTS_DIR=${WORKSPACE}/testresults

conda/recipes/blazingsql-build-env/meta.yaml

@@ -0,0 +1,57 @@
+{% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set py_version = environ.get('CONDA_PY', 36) %}
+
+### 
+# Versions referenced below are set in `conda/recipe/*versions.yaml` except for
+#   those set above (e.g. `cuda_version`)
+#
+# gpuCI loads the correct file based on the build type (NIGHTLY or RELEASE) in 
+#   `ci/cpu/build-env.sh` and `ci/axis/*.yaml`
+#
+# Manual builds need to use the `conda build` flag of `-m <path-to-file>.yaml` 
+#   to set these versions
+###
+
+package:
+    name: blazingsql-build-env
+    version: {{ rapids_version }}
+
+source:
+    path: ../../..
+
+build:
+  number: {{ GIT_DESCRIBE_NUMBER }}
+  string: cuda{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  script_env:
+    - CUDA_VERSION
+    - RAPIDS_VER
+    - VERSION_SUFFIX
+
+requirements:
+    build:
+        - python
+    run:
+        - aws-sdk-cpp
+        - cmake {{ cmake_version }}
+        - cppzmq
+        - cudatoolkit {{ cuda_version }}.*
+        - cython {{ cython_version }}
+        - dlpack {{ dlpack_version }}
+        - google-cloud-cpp {{ google_cloud_cpp_version }}
+        - gtest {{ gtest_version }}
+        - jpype1
+        - maven
+        - netifaces
+        - ninja
+        - openjdk {{ openjdk_version }}
+        - pyhive
+        - python
+        - setuptools
+        - spdlog {{ spdlog_version }}
+        - zeromq
+
+about:
+    home: http://www.blazingsql.com/
+    license: Apache-2.0
+    license_family: Apache

conda/recipes/blazingsql-notebook-env/meta.yaml

@@ -0,0 +1,63 @@
+{% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %}
+{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %}
+{% set py_version = environ.get('CONDA_PY', 36) %}
+
+### 
+# Versions referenced below are set in `conda/recipe/*versions.yaml` except for
+#   those set above (e.g. `cuda_version`)
+#
+# gpuCI loads the correct file based on the build type (NIGHTLY or RELEASE) in 
+#   `ci/cpu/build-env.sh` and `ci/axis/*.yaml`
+#
+# Manual builds need to use the `conda build` flag of `-m <path-to-file>.yaml` 
+#   to set these versions
+###
+
+package:
+    name: blazingsql-notebook-env
+    version: {{ rapids_version }}
+
+source:
+    path: ../../..
+
+build:
+  number: {{ GIT_DESCRIBE_NUMBER }}
+  string: cuda{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  script_env:
+    - CUDA_VERSION
+    - RAPIDS_VER
+    - VERSION_SUFFIX
+
+requirements:
+  build:
+    - python
+  run:
+    - colorcet
+    - cudatoolkit {{ cuda_version }}
+    - dash
+    - dask
+    - dask-ml
+    - datashader
+    - fbprophet
+    - geoviews
+    - holoviews
+    - ipywidgets
+    - jupyter-dash
+    - matplotlib
+    - networkx
+    - panel {{ panel_version }}
+    - plotly
+    - python-graphviz
+    - pytorch {{ pytorch_version }}
+    - s3fs {{ s3fs_version }}
+    - scikit-learn
+    - scipy
+    - seaborn
+    - spacy
+    - statsmodels
+    - xgboost
+
+about:
+    home: http://www.blazingsql.com/
+    license: Apache-2.0
+    license_family: Apache

conda/recipes/rapids-blazing/LICENSE

@@ -0,0 +1 @@
+The license of this package is a combination of the dependent packages contained herein.