[RELEASE] integration v21.10

ci/axis/nightly-arm64.yaml

@@ -0,0 +1,20 @@
+CONDA_USERNAME:
+  - rapidsai-nightly
+
+CONDA_CONFIG_FILE:
+  - conda/recipes/versions.yaml
+
+# Use M.X.Ya (major.minor.patch) with 'a' version to match nightly tag
+RAPIDS_VER:
+  - 21.10.00a
+
+# Use CUDA_VER to not clobber `CUDA_VERSION` in the container
+CUDA_VER:
+  - 11.4
+  - 11.2
+
+PYTHON_VER:
+  - 3.7
+  - 3.8
+
+exclude:

ci/axis/nightly.yaml

@@ -6,7 +6,7 @@ CONDA_CONFIG_FILE:
 
 # Use M.X.Ya (major.minor.patch) with 'a' version to match nightly tag
 RAPIDS_VER:
-  - 21.08.00a
+  - 21.10.00a
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
 CUDA_VER:

ci/axis/release.yaml

@@ -6,7 +6,7 @@ CONDA_CONFIG_FILE:
 
 # Use M.X.Y (major.minor.patch) version to match tag
 RAPIDS_VER:
-  - "21.08.00"
+  - "21.10.00"
 
 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container
 CUDA_VER:

ci/axis/tests.yaml

@@ -6,9 +6,10 @@ DOCKER_REPO:
 
 # Use M.X (major.minor) version
 RAPIDS_VER:
-  - "21.08"
+  - "21.10"
 
 CUDA_VER:
+  - 11.4
   - 11.2
   - 11.0
 

ci/cpu/build.sh

@@ -49,6 +49,9 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
 fi
 
+# Get arch
+ARCH=$(uname -m)
+
 function build_pkg {
   # Build pkg
   gpuci_logger "Start conda build for '${1}'..."
@@ -72,7 +75,6 @@ function run_builds {
 function upload_builds {
 
   # Check arch
-  ARCH=$(uname -m)
   if [ "${ARCH}" = "x86_64" ]; then
     ARCH_DIR="linux-64"
   elif [ "${ARCH}" = "aarch64" ]; then
@@ -118,9 +120,12 @@ if [[ "$BUILD_PKGS" == "env" || -z "$BUILD_PKGS" ]] ; then
   # Run builds for env-pkgs
   run_builds $CONDA_RAPIDS_BUILD_RECIPE
   run_builds $CONDA_RAPIDS_NOTEBOOK_RECIPE
-  run_builds $CONDA_RAPIDS_DOC_RECIPE
-  run_builds $CONDA_BLAZING_BUILD_RECIPE
-  run_builds $CONDA_BLAZING_NOTEBOOK_RECIPE
+  # Bypass not supported packages for arm64
+  if [ "${ARCH}" != "aarch64" ]; then
+    run_builds $CONDA_RAPIDS_DOC_RECIPE
+    run_builds $CONDA_BLAZING_BUILD_RECIPE
+    run_builds $CONDA_BLAZING_NOTEBOOK_RECIPE
+  fi
 fi
 
 # Upload builds

ci/test/cudf.sh

@@ -26,32 +26,24 @@ for gt in /rapids/cudf/cpp/build/gtests/*; do
    fi
 done
 
-# Python tests
-export PYTHONPATH=\
-/rapids/cudf/python/cudf:\
-/rapids/cudf/python/dask_cudf:\
-/rapids/cudf/python/custreamz:\
-/rapids/cudf/python/nvstrings:\
-${PYTHONPATH}
-
-cd /rapids/cudf/python/cudf
-py.test -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-cudf.xml -v
+cd /rapids/cudf/python/cudf/cudf/tests
+pytest -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-cudf.xml -v
 exitcode=$?
 if (( ${exitcode} != 0 )); then
    SUITEERROR=${exitcode}
    echo "FAILED: 1 or more python tests"
 fi
 
 cd /rapids/cudf/python/dask_cudf
-py.test -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-dask-cudf.xml -v
+pytest -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-dask-cudf.xml -v
 exitcode=$?
 if (( ${exitcode} != 0 )); then
    SUITEERROR=${exitcode}
    echo "FAILED: 1 or more python tests"
 fi
 
 cd /rapids/cudf/python/custreamz
-py.test -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-custreamz.xml -v
+pytest -n 6 --junitxml=${TESTRESULTS_DIR}/pytest-custreamz.xml -v
 exitcode=$?
 if (( ${exitcode} != 0 )); then
    SUITEERROR=${exitcode}

ci/test/cugraph.sh

@@ -1,47 +1,12 @@
 #!/bin/bash
 set +e
 set -x
+
 export HOME="${WORKSPACE}"
 export LIBCUDF_KERNEL_CACHE_PATH="${WORKSPACE}/.jitcache"
 export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/local/gcc7/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-# FIXME: "source activate" line should not be needed
 source /opt/conda/bin/activate rapids
 
-# Get datasets
-cd /rapids/cugraph/datasets
-bash ./get_test_data.sh
-export RAPIDS_DATASET_ROOT_DIR=/rapids/cugraph/datasets
-
-# Show environment
-env
-nvidia-smi
-conda list
-
-# Install pytest plugin for cugraph
-conda install rapids-pytest-benchmark
-
-TESTRESULTS_DIR="${WORKSPACE}/testresults"
-mkdir -p ${TESTRESULTS_DIR}
-SUITEERROR=0
-
-# gtests
-for gt in /rapids/cugraph/cpp/build/tests/*_TEST; do
-   # FIXME: remove this ASAP
-   ${gt} --gtest_output=xml:${TESTRESULTS_DIR}/
-   exitcode=$?
-   if (( ${exitcode} != 0 )); then
-      SUITEERROR=${exitcode}
-      echo "FAILED: ${gt}"
-   fi
-done
-
-# Python tests
-pytest --ignore=/rapids/cugraph/python/cugraph/raft --junitxml=${TESTRESULTS_DIR}/pytest.xml -v /rapids/cugraph/python
-exitcode=$?
-if (( ${exitcode} != 0 )); then
-   SUITEERROR=${exitcode}
-   echo "FAILED: 1 or more tests in /rapids/cugraph/python"
-fi
-
-exit ${SUITEERROR}
+cd /rapids/cugraph
+bash /rapids/cugraph/ci/test.sh

conda/recipes/rapids-build-env/meta.yaml

@@ -116,7 +116,7 @@ requirements:
     - pydocstyle {{ pydocstyle_version }}
     - pynvml
     - pyorc
-    # - pyppeteer {{ pyppeteer_version }}
+    - pyppeteer {{ pyppeteer_version }}
     - pyproj {{ pyproj_version }}
     - pytest
     - pytest-asyncio {{ pytest_asyncio_version }}
@@ -127,6 +127,7 @@ requirements:
     - python
     - python-confluent-kafka {{ python_confluent_kafka_version }}
     - python-louvain
+    - python-snappy
     - rapidjson {{ rapidjson_version }}
     - ripgrep
     - s3fs {{ s3fs_version }}
@@ -138,7 +139,7 @@ requirements:
     - spdlog {{ spdlog_version }}
     - statsmodels
     - streamz
-    - transformers
+    - transformers {{ transformers_version }}
     - treelite {{ treelite_version }}
     - twine
     - typing_extensions

conda/recipes/rapids-doc-env/meta.yaml

@@ -42,6 +42,7 @@ requirements:
     - nbsphinx {{ nbsphinx_version }}
     - numpydoc
     - pandoc {{ pandoc_version }}
+    - pydata-sphinx-theme {{ pydata_sphinx_theme_version }}
     - recommonmark
     - sphinx
     - sphinx_rtd_theme

conda/recipes/rapids-notebook-env/meta.yaml

@@ -51,6 +51,7 @@ requirements:
     - ipywidgets
     - jupyter-server-proxy
     - jupyterlab {{ jupyterlab_version }}
+    - jupyterlab-favorites
     - jupyterlab-nvdashboard
     - jupyter-packaging {{ jupyter_packaging_version }}
     - jupyterlab_widgets

conda/recipes/rapids/meta.yaml

@@ -57,13 +57,13 @@ requirements:
     - rapids-xgboost ={{ minor_version }}.*
     - rmm ={{ minor_version }}.*
 
-test:
-  imports:
-    - cudf
-    - dask_cudf
-    - cuml
-    - cugraph
-    - cuspatial
+test:            # [linux64]
+  imports:       # [linux64]
+    - cudf       # [linux64]
+    - dask_cudf  # [linux64]
+    - cuml       # [lunux64]
+    - cugraph    # [linux64]
+    - cuspatial  # [linux64]
 
 about:
   home: http://rapids.ai/

conda/recipes/versions.yaml

@@ -21,11 +21,11 @@ conda_verify_version:
 
 # Versions for `libgcc-ng`, `libgfortran-ng`, `libstdcxx-ng` stack
 build_stack_version:
-  - '=9.3.0'
+  - '=9.4.0'
 
 # Shared versions across meta-pkgs
 arrow_version:
-  - '=4.0.1'
+  - '=5.0.0'
 benchmark_version:
   - '=1.5.1'
 black_version:
@@ -47,11 +47,11 @@ cupy_version:
 cython_version:
   - '>=0.29.17,<0.30'
 dask_version:
-  - '>=2021.6.0,<=2021.07.1'
+  - '=2021.9.1'
 datashader_version:
   - '>=0.11.1,<0.12'
 distributed_version:
-  - '>=2021.6.0,<=2021.07.1'
+  - '=2021.9.1'
 dlpack_version:
   - '>=0.5,<0.6.0a0'
 double_conversion_version:
@@ -67,13 +67,13 @@ flake8_version:
 fsspec_version:
   - '>=0.9.0'
 gdal_version:
-  - '>=3.2.0,<3.3.0a0'
+  - '>=3.3.0,<3.4.0a0'
 geopandas_version:
   - '>=0.9.0,<0.10.0a0'
 gcsfs_version:
   - '>=0.8.0'
 google_cloud_cpp_version:
-  - '>=1.25.0'
+  - '>=1.25.0,<1.30'
 gmock_version:
   - '=1.10.0'
 gtest_version:
@@ -115,7 +115,7 @@ nvtx_version:
 openjdk_version:
   - '>=8.0'
 pandas_version:
-  - '>=1.0,<1.3.0dev0'
+  - '>=1.0,<1.4.0dev0'
 pandoc_version:
   - '<=2.0.0'
 panel_version:
@@ -130,10 +130,12 @@ pytorch_version:
   - '>=1.6'
 protobuf_version:
   - '>=3.4.1,<4.0.0'
+pydata_sphinx_theme_version:
+  - '>=0.6.3'
 pyproj_version:
   - '>=2.4,<=3.1'
 pyppeteer_version:
-  - '<=0.2.2'
+  - '>=0.2.6'
 pytest_asyncio_version:
   - '<0.14.0'
 rapidjson_version:
@@ -143,7 +145,7 @@ s3fs_version:
 scikit_image_version:
   - '>=0.18.0,<0.19.0'
 scikit_learn_version:
-  - '=0.23.1'
+  - '=0.24'
 scipy_version:
   - '=1.6.0'
 setuptools_version:
@@ -153,4 +155,6 @@ spdlog_version:
 sphinx_markdown_tables_version:
   - '=0.0.14=pyh9f0ad1d_1'
 treelite_version:
-  - '=2.0.0'
+  - '=2.1.0'
+transformers_version:
+  - '<=4.10.3'