diff --git a/ci/axis/nightly.yaml b/ci/axis/nightly.yaml index 90370d7b..3d50d9bc 100644 --- a/ci/axis/nightly.yaml +++ b/ci/axis/nightly.yaml @@ -4,8 +4,8 @@ CONDA_USERNAME: CONDA_CONFIG_FILE: - conda/recipes/versions.yaml +# Use M.X.Ya (major.minor.patch) with 'a' version to match nightly tag RAPIDS_VER: - - 0.16.0a - 0.17.0a # Use CUDA_VER to not clobber `CUDA_VERSION` in the container @@ -14,6 +14,7 @@ CUDA_VER: - 10.2 - 10.1 +# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated DEFAULT_CUDA_VER: - 10.1 diff --git a/ci/axis/release.yaml b/ci/axis/release.yaml index 66f75b45..b715d458 100644 --- a/ci/axis/release.yaml +++ b/ci/axis/release.yaml @@ -4,8 +4,9 @@ CONDA_USERNAME: CONDA_CONFIG_FILE: - conda/recipes/versions.yaml +# Use M.X.Y (major.minor.patch) version to match tag RAPIDS_VER: - - 0.16 + - 0.17.0 # Use CUDA_VER to not clobber `CUDA_VERSION` in the container CUDA_VER: @@ -13,6 +14,7 @@ CUDA_VER: - 10.2 - 10.1 +# TODO: DEPRECATED - No longer used and needs to be removed & gpuCI jobs updated DEFAULT_CUDA_VER: - 10.1 diff --git a/ci/axis/tests.yaml b/ci/axis/tests.yaml index 7728be16..6e1f66fb 100644 --- a/ci/axis/tests.yaml +++ b/ci/axis/tests.yaml @@ -4,8 +4,9 @@ RUNTIME_DOCKER_REPO: DOCKER_REPO: - rapidsai/rapidsai-dev-nightly +# Use M.X (major.minor) version RAPIDS_VER: - - 0.16 + - 0.17 CUDA_VER: - 11.0 diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index c63c4f79..3dc8a665 100644 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -16,15 +16,15 @@ set -e export PATH=/conda/bin:$PATH export HOME=$WORKSPACE -# Save original build offset -export ORIG_OFFSET=$RAPIDS_OFFSET - # Set recipe paths CONDA_XGBOOST_RECIPE="conda/recipes/rapids-xgboost" CONDA_RAPIDS_RECIPE="conda/recipes/rapids" +CONDA_RAPIDS_BLAZING_RECIPE="conda/recipes/rapids-blazing" CONDA_RAPIDS_BUILD_RECIPE="conda/recipes/rapids-build-env" CONDA_RAPIDS_NOTEBOOK_RECIPE="conda/recipes/rapids-notebook-env" CONDA_RAPIDS_DOC_RECIPE="conda/recipes/rapids-doc-env" +CONDA_BLAZING_BUILD_RECIPE="conda/recipes/blazingsql-build-env" +CONDA_BLAZING_NOTEBOOK_RECIPE="conda/recipes/blazingsql-notebook-env" # Allow insecure connections for conda-mirror echo "ssl_verify: False" >> /conda/.condarc @@ -50,40 +50,23 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then fi function build_pkg { - # Create 'conda_build_config.yaml' with CUDA version - gpuci_logger "Creating 'conda_build_config.yaml' with CUDA_VER='$CUDA_VER' to select correct pkgs..." - cat > ${1}/conda_build_config.yaml <=1.4" ;; *) @@ -21,6 +21,9 @@ esac env +nvidia-smi +conda list + /test.sh 2>&1 | tee nbtest.log EXITCODE=$? python /rapids/utils/nbtestlog2junitxml.py nbtest.log diff --git a/ci/test/rmm.sh b/ci/test/rmm.sh index 93c5b76f..865c2470 100644 --- a/ci/test/rmm.sh +++ b/ci/test/rmm.sh @@ -6,6 +6,7 @@ export PATH="/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local # FIXME: "source activate" line should not be needed source /opt/conda/bin/activate rapids env +nvidia-smi conda list TESTRESULTS_DIR=${WORKSPACE}/testresults diff --git a/conda/recipes/blazingsql-build-env/meta.yaml b/conda/recipes/blazingsql-build-env/meta.yaml new file mode 100644 index 00000000..88b6c210 --- /dev/null +++ b/conda/recipes/blazingsql-build-env/meta.yaml @@ -0,0 +1,57 @@ +{% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %} +{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %} +{% set py_version = environ.get('CONDA_PY', 36) %} + +### +# Versions referenced below are set in `conda/recipe/*versions.yaml` except for +# those set above (e.g. `cuda_version`) +# +# gpuCI loads the correct file based on the build type (NIGHTLY or RELEASE) in +# `ci/cpu/build-env.sh` and `ci/axis/*.yaml` +# +# Manual builds need to use the `conda build` flag of `-m .yaml` +# to set these versions +### + +package: + name: blazingsql-build-env + version: {{ rapids_version }} + +source: + path: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CUDA_VERSION + - RAPIDS_VER + - VERSION_SUFFIX + +requirements: + build: + - python + run: + - aws-sdk-cpp + - cmake {{ cmake_version }} + - cppzmq + - cudatoolkit {{ cuda_version }}.* + - cython {{ cython_version }} + - dlpack {{ dlpack_version }} + - google-cloud-cpp {{ google_cloud_cpp_version }} + - gtest {{ gtest_version }} + - jpype1 + - maven + - netifaces + - ninja + - openjdk {{ openjdk_version }} + - pyhive + - python + - setuptools + - spdlog {{ spdlog_version }} + - zeromq + +about: + home: http://www.blazingsql.com/ + license: Apache-2.0 + license_family: Apache diff --git a/conda/recipes/blazingsql-notebook-env/meta.yaml b/conda/recipes/blazingsql-notebook-env/meta.yaml new file mode 100644 index 00000000..46ba37ed --- /dev/null +++ b/conda/recipes/blazingsql-notebook-env/meta.yaml @@ -0,0 +1,63 @@ +{% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %} +{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %} +{% set py_version = environ.get('CONDA_PY', 36) %} + +### +# Versions referenced below are set in `conda/recipe/*versions.yaml` except for +# those set above (e.g. `cuda_version`) +# +# gpuCI loads the correct file based on the build type (NIGHTLY or RELEASE) in +# `ci/cpu/build-env.sh` and `ci/axis/*.yaml` +# +# Manual builds need to use the `conda build` flag of `-m .yaml` +# to set these versions +### + +package: + name: blazingsql-notebook-env + version: {{ rapids_version }} + +source: + path: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CUDA_VERSION + - RAPIDS_VER + - VERSION_SUFFIX + +requirements: + build: + - python + run: + - colorcet + - cudatoolkit {{ cuda_version }} + - dash + - dask + - dask-ml + - datashader + - fbprophet + - geoviews + - holoviews + - ipywidgets + - jupyter-dash + - matplotlib + - networkx + - panel {{ panel_version }} + - plotly + - python-graphviz + - pytorch {{ pytorch_version }} + - s3fs {{ s3fs_version }} + - scikit-learn + - scipy + - seaborn + - spacy + - statsmodels + - xgboost + +about: + home: http://www.blazingsql.com/ + license: Apache-2.0 + license_family: Apache diff --git a/conda/recipes/rapids-blazing/LICENSE b/conda/recipes/rapids-blazing/LICENSE new file mode 100644 index 00000000..d00c6281 --- /dev/null +++ b/conda/recipes/rapids-blazing/LICENSE @@ -0,0 +1 @@ +The license of this package is a combination of the dependent packages contained herein. diff --git a/conda/recipes/rapids-blazing/meta.yaml b/conda/recipes/rapids-blazing/meta.yaml new file mode 100644 index 00000000..eb10a0b2 --- /dev/null +++ b/conda/recipes/rapids-blazing/meta.yaml @@ -0,0 +1,61 @@ +{% set rapids_version = environ.get('RAPIDS_VER', '0.0.0') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = rapids_version.split('.')[0] + '.' + rapids_version.split('.')[1] %} +{% set cuda_version = '.'.join(environ.get('CUDA_VERSION', '10.0').split('.')[:2]) %} +{% set py_version = environ.get('CONDA_PY', 36) %} + +### +# Versions referenced below are set in `conda/recipe/*versions.yaml` except for +# those set above (e.g. `cuda_version`) +# +# gpuCI loads the correct file based on the build type (NIGHTLY or RELEASE) in +# `ci/cpu/build-meta.sh` and `ci/axis/*.yaml` +# +# Manual builds need to use the `conda build` flag of `-m .yaml` +# to set these versions +### + +package: + name: rapids-blazing + version: {{ rapids_version }} + +source: + path: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_version }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CUDA_VERSION + - RAPIDS_VER + - VERSION_SUFFIX + +requirements: + host: + - python + run: + - cudatoolkit ={{ cuda_version }}.* + - rapids ={{ minor_version }}.* + - blazingsql ={{ minor_version }}.* + +test: + requires: + - cudatoolkit {{ cuda_version }}.* + imports: + - cio + - cudf + - dask_cudf + - cuml + - cugraph + - cuspatial + +about: + home: http://rapids.ai/ + license: Custom + license_file: conda/recipes/rapids/LICENSE + summary: 'RAPIDS Suite - Open GPU Data Science' + description: | + Meta-package for the RAPIDS suite of software libraries. RAPIDS gives you the freedom to execute end-to-end data science + and analytics pipelines entirely on GPUs. It relies on NVIDIA® CUDA® primitives for low-level compute optimization, + but exposes that GPU parallelism and high-bandwidth memory speed through user-friendly Python interfaces. + doc_url: https://docs.rapids.ai/ + dev_url: https://github.com/rapidsai/ diff --git a/conda/recipes/rapids-build-env/meta.yaml b/conda/recipes/rapids-build-env/meta.yaml index 0ad6a327..d9e270ae 100644 --- a/conda/recipes/rapids-build-env/meta.yaml +++ b/conda/recipes/rapids-build-env/meta.yaml @@ -37,6 +37,7 @@ requirements: run: - arrow-cpp {{ arrow_version }} - arrow-cpp-proc * cuda + - asvdb - autoconf - automake - benchmark {{ benchmark_version }} @@ -48,6 +49,7 @@ requirements: - conda-forge::clang {{ clang_version }} - conda-forge::clang-tools {{ clang_version }} - cmake {{ cmake_version }} + - cmake-format {{ cmake_format_version }} - cmake_setuptools {{ cmake_setuptools_version }} - conda {{ conda_version }} - conda-build {{ conda_build_version }} @@ -108,10 +110,10 @@ requirements: - pyarrow {{ arrow_version }} - pydeck {{ pydeck_version }} - pynvml - - pyppeteer + - pyppeteer {{ pyppeteer_version }} - pyproj {{ pyproj_version }} - pytest - - pytest-asyncio + - pytest-asyncio {{ pytest_asyncio_version }} - pytest-benchmark - pytest-cov - pytest-timeout @@ -119,6 +121,7 @@ requirements: - python-confluent-kafka {{ python_confluent_kafka_version }} - python-louvain - rapidjson {{ rapidjson_version }} + - rapids-pytest-benchmark - ripgrep - s3fs - setuptools diff --git a/conda/recipes/rapids-notebook-env/meta.yaml b/conda/recipes/rapids-notebook-env/meta.yaml index 279f8be3..70fda447 100644 --- a/conda/recipes/rapids-notebook-env/meta.yaml +++ b/conda/recipes/rapids-notebook-env/meta.yaml @@ -36,12 +36,15 @@ requirements: - python run: - bokeh {{ bokeh_version }} + - colorcet - conda-forge::blas - cudatoolkit ={{ cuda_version }}.* - cupy {{ cupy_version }} - cython {{ cython_version }} - dask-labextension - dask-ml + - datashader {{ datashader_version }} + - fa2 - filterpy - holoviews - ipython {{ ipython_version }} diff --git a/conda/recipes/versions.yaml b/conda/recipes/versions.yaml index ddf98aee..23045e34 100644 --- a/conda/recipes/versions.yaml +++ b/conda/recipes/versions.yaml @@ -8,7 +8,7 @@ dask_xgboost_version: # Versions for `rapids-xgboost` meta-pkg xgboost_version: - - '=1.2.0dev.rapidsai0.16' + - '=1.3.0dev.rapidsai0.17' # Versions for conda conda_version: @@ -30,13 +30,15 @@ benchmark_version: black_version: - '=19.10' bokeh_version: - - '>=2.1.1' + - '>=2.1.1,<=2.2.3' boost_cpp_version: - '=1.72.0' clang_version: - '=8.0.1' cmake_version: - - '=3.17' + - '=3.18' +cmake_format_version: + - '=0.6.11' cmake_setuptools_version: - '>=0.1.3' cupy_version: @@ -46,7 +48,7 @@ cython_version: dask_version: - '>=2.23.0' datashader_version: - - '>=0.11.1' + - '>=0.11.1,<0.12' distributed_version: - '>=2.23.0' dlpack_version: @@ -68,7 +70,9 @@ fsspec_version: gdal_version: - '>=3.0.2,<3.1.0a0' geopandas_version: - - '>=0.6' + - '>=0.6, <=0.8.1' +google_cloud_cpp_version: + - '=1.16.0' gmock_version: - '=1.10.0' gtest_version: @@ -80,7 +84,7 @@ isort_version: jupyterlab_version: - '=2.1' librdkafka_version: - - '=1.5.0' + - '=1.5.*' mypy_version: - '0.782' nccl_version: @@ -95,22 +99,32 @@ numpy_version: - '>=1.17.3' nvtx_version: - '>=0.2.1,<0.3' +openjdk_version: + - '>=8.0' pandas_version: - '>=1.0,<1.2.0dev0' pandoc_version: - '<=2.0.0' panel_version: - - '>=0.9' + - '>=0.9.1, <=0.9.7' pydeck_version: - - '>=0.3' + - '>=0.3, <=0.5.0' python_confluent_kafka_version: - '>=1.3.0' +pytorch_version: + - '>=1.6' protobuf_version: - '>=3.4.1,<4.0.0' pyproj_version: - - '>=2.4' + - '>=2.4,<=2.6.1.post1' +pyppeteer_version: + - '<=0.2.2' +pytest_asyncio_version: + - '<0.14.0' rapidjson_version: - '=1.1.0' +s3fs_version: + - '>=0.5.1' scikit_learn_version: - '=0.23.1' scipy_version: