From f4572f50edea998723aacf1cf648de8a476ed362 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 6 Nov 2020 11:47:39 -0800 Subject: [PATCH 1/6] Gpuciscripts clean and update --- ci/checks/style.sh | 7 +++-- ci/cpu/build.sh | 33 +++++++++++++--------- ci/cpu/upload-anaconda.sh | 2 +- ci/gpu/build.sh | 53 ++++++++++++++++++++---------------- ci/local/README.md | 3 +- ci/release/update-version.sh | 2 +- 6 files changed, 58 insertions(+), 42 deletions(-) diff --git a/ci/checks/style.sh b/ci/checks/style.sh index 965f62926..ab026e5aa 100755 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -1,15 +1,16 @@ #!/bin/bash -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2020, NVIDIA CORPORATION. ################################################################################ # dask-cuda Style Tester ################################################################################ # Ignore errors and set path set +e -PATH=/conda/bin:$PATH +PATH=/opt/conda/bin:$PATH # Activate common conda env -source activate gdf +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids # Run isort and get results/return code ISORT=`isort --recursive --check-only .` diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 8d308332c..5091e92f2 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2020, NVIDIA CORPORATION. ################################################################################ # dask-cuda cpu build ################################################################################ set -e # Logger function for build status output -function logger() { +function gpuci_logger() { echo -e "\n>>>> $@\n" } # Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH # Set home to the job's workspace export HOME=$WORKSPACE @@ -23,21 +23,28 @@ cd $WORKSPACE export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` +# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) +export GPUCI_CONDA_RETRY_MAX=1 +export GPUCI_CONDA_RETRY_SLEEP=30 + ################################################################################ # SETUP - Check environment ################################################################################ -logger "Get env..." +gpuci_logger "Get env" env -logger "Activate conda env..." -source activate gdf +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids -logger "Check versions..." +gpuci_logger "Check versions" python --version -gcc --version -g++ --version -conda list +$CC --version +$CXX --version +conda info +conda config --show-sources +conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False @@ -53,7 +60,7 @@ fi # BUILD - Package builds ################################################################################ -conda build conda/recipes/dask-cuda --python=${PYTHON} +gpuci_conda_retry build conda/recipes/dask-cuda --python=${PYTHON} rm -rf dist/ python setup.py sdist bdist_wheel @@ -62,8 +69,8 @@ python setup.py sdist bdist_wheel # UPLOAD - Packages ################################################################################ -logger "Upload conda pkg..." +gpuci_logger "Upload conda pkg" source ci/cpu/upload-anaconda.sh -logger "Upload pypi pkg..." +gpuci_logger "Upload pypi pkg" source ci/cpu/upload-pypi.sh \ No newline at end of file diff --git a/ci/cpu/upload-anaconda.sh b/ci/cpu/upload-anaconda.sh index cf6c34f90..361af0490 100755 --- a/ci/cpu/upload-anaconda.sh +++ b/ci/cpu/upload-anaconda.sh @@ -4,7 +4,7 @@ set -e -export UPLOADFILE=`conda build conda/recipes/dask-cuda --python=$PYTHON --output` +export UPLOADFILE=`gpuci_conda_retry build conda/recipes/dask-cuda --python=$PYTHON --output` CUDA_REL=${CUDA_VERSION%.*} diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index d80f2a391..209c6a80d 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright (c) 2020, NVIDIA CORPORATION. ############################################## # dask-cuda GPU build and test script for CI # ############################################## @@ -8,7 +8,7 @@ NUMARGS=$# ARGS=$* # Logger function for build status output -function logger() { +function gpuci_logger() { echo -e "\n>>>> $@\n" } @@ -18,8 +18,8 @@ function hasArg { } # Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=4 +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=-4 export CUDA_REL=${CUDA_VERSION%.*} export CUDA_REL2=${CUDA//./} @@ -44,53 +44,60 @@ export INSTALL_DASK_MASTER=1 # SETUP - Check environment ################################################################################ -logger "Check environment..." +gpuci_logger "Check environment" env -logger "Check GPU usage..." +gpuci_logger "Check GPU usage" nvidia-smi -logger "Activate conda env..." -source activate gdf -conda list +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids +conda info +conda config --show-sources +conda list --show-channel-urls # Fixing Numpy version to avoid RuntimeWarning: numpy.ufunc size changed, may # indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject -conda install "cudatoolkit=$CUDA_REL" \ +gpuci_conda_retry install "cudatoolkit=$CUDA_REL" \ "cudf=${MINOR_VERSION}" "dask-cudf=${MINOR_VERSION}" \ "ucx-py=$MINOR_VERSION.*" "ucx-proc=*=gpu" \ "rapids-build-env=$MINOR_VERSION.*" # Pin pytest-asyncio because latest versions modify the default asyncio # `event_loop_policy`. See https://github.com/dask/distributed/pull/4212 . -conda install "pytest-asyncio=<0.14.0" +gpuci_conda_retry install "pytest-asyncio=<0.14.0" # https://docs.rapids.ai/maintainers/depmgmt/ -# conda remove -f rapids-build-env -# conda install "your-pkg=1.0.0" +# gpuci_conda_retry remove -f rapids-build-env +# gpuci_conda_retry install "your-pkg=1.0.0" -conda list +conda info +conda config --show-sources +conda list --show-channel-urls # Install the master version of dask and distributed if [[ "${INSTALL_DASK_MASTER}" == 1 ]]; then - logger "pip install git+https://github.com/dask/distributed.git --upgrade" + gpuci_logger "pip install git+https://github.com/dask/distributed.git --upgrade" pip install "git+https://github.com/dask/distributed.git" --upgrade - logger "pip install git+https://github.com/dask/dask.git --upgrade" + gpuci_logger "pip install git+https://github.com/dask/dask.git --upgrade" pip install "git+https://github.com/dask/dask.git" --upgrade fi -logger "Check versions..." +gpuci_logger "Check versions" python --version $CC --version $CXX --version -conda list +conda info +conda config --show-sources +conda list --show-channel-urls ################################################################################ # BUILD - Build dask-cuda ################################################################################ -logger "Build dask-cuda..." +gpuci_logger "Build dask-cuda" cd $WORKSPACE python -m pip install -e . @@ -99,17 +106,17 @@ python -m pip install -e . ################################################################################ if hasArg --skip-tests; then - logger "Skipping Tests..." + gpuci_logger "Skipping Tests" else - logger "Python py.test for dask-cuda..." + gpuci_logger "Python py.test for dask-cuda" cd $WORKSPACE ls dask_cuda/tests/ UCXPY_IFNAME=eth0 UCX_WARN_UNUSED_ENV_VARS=n UCX_MEMTYPE_CACHE=n py.test -vs --cache-clear --basetemp=${WORKSPACE}/dask-cuda-tmp --junitxml=${WORKSPACE}/junit-dask-cuda.xml --cov-config=.coveragerc --cov=dask_cuda --cov-report=xml:${WORKSPACE}/dask-cuda-coverage.xml --cov-report term dask_cuda/tests/ - logger "Running dask.distributed GPU tests" + gpuci_logger "Running dask.distributed GPU tests" # Test downstream packages, which requires Python v3.7 if [ $(python -c "import sys; print(sys.version_info[1])") -ge "7" ]; then - logger "TEST OF DASK/UCX..." + gpuci_logger "TEST OF DASK/UCX" py.test --cache-clear -vs `python -c "import distributed.protocol.tests.test_cupy as m;print(m.__file__)"` py.test --cache-clear -vs `python -c "import distributed.protocol.tests.test_numba as m;print(m.__file__)"` py.test --cache-clear -vs `python -c "import distributed.protocol.tests.test_rmm as m;print(m.__file__)"` diff --git a/ci/local/README.md b/ci/local/README.md index 3dc6e0f51..b121cfc05 100644 --- a/ci/local/README.md +++ b/ci/local/README.md @@ -30,7 +30,8 @@ For a full list of available gpuCI docker images, visit our [DockerHub](https:// Style Check: ```bash $ bash ci/local/build.sh -r ~/rapids/dask-cuda -s -$ source activate gdf #Activate gpuCI conda environment +$ . /opt/conda/etc/profile.d/conda.sh +$ conda activate rapids #Activate gpuCI conda environment $ cd rapids $ flake8 python ``` diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index e08539083..7e5a25286 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2020, NVIDIA CORPORATION. ################################################################################ # dask-cuda version updater ################################################################################ From c36a65843a592d36f256f8451b91b20f8320cb1d Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Thu, 12 Nov 2020 20:59:42 -0800 Subject: [PATCH 2/6] Updated upload.sh and build.sh --- ci/cpu/build.sh | 46 ++++++++++++++++++--------------------- ci/cpu/upload-anaconda.sh | 27 ----------------------- ci/cpu/upload.sh | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 52 deletions(-) delete mode 100755 ci/cpu/upload-anaconda.sh create mode 100755 ci/cpu/upload.sh diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 5091e92f2..674b1e59a 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -1,47 +1,49 @@ #!/usr/bin/env bash -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2019, NVIDIA CORPORATION. ################################################################################ # dask-cuda cpu build ################################################################################ set -e -# Logger function for build status output -function gpuci_logger() { - echo -e "\n>>>> $@\n" -} - # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} # Set home to the job's workspace export HOME=$WORKSPACE -# Switch to project root; also root of repo checkout -cd $WORKSPACE - -# Get latest tag and number of commits since tag -export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` -export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` +# Determine CUDA release version +export CUDA_REL=${CUDA_VERSION%.*} # Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) export GPUCI_CONDA_RETRY_MAX=1 export GPUCI_CONDA_RETRY_SLEEP=30 +# Switch to project root; also root of repo checkout +cd $WORKSPACE + +# If nightly build, append current YYMMDD to version +if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then + export VERSION_SUFFIX=`date +%y%m%d` +fi + ################################################################################ # SETUP - Check environment ################################################################################ -gpuci_logger "Get env" +gpuci_logger "Check environment variables" env gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -gpuci_logger "Check versions" +gpuci_logger "Check compiler versions" python --version $CC --version $CXX --version + +gpuci_logger "Check conda environment" conda info conda config --show-sources conda list --show-channel-urls @@ -49,17 +51,11 @@ conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False -pip install git+https://github.com/dask/distributed.git@master - -# If nightly build, append current YYMMDD to version -if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then - export VERSION_SUFFIX=`date +%y%m%d` -fi - ################################################################################ # BUILD - Package builds ################################################################################ +gpuci_logger "Build conda pkg for libcudf" gpuci_conda_retry build conda/recipes/dask-cuda --python=${PYTHON} rm -rf dist/ @@ -69,8 +65,8 @@ python setup.py sdist bdist_wheel # UPLOAD - Packages ################################################################################ -gpuci_logger "Upload conda pkg" -source ci/cpu/upload-anaconda.sh +gpuci_logger "Upload conda pkg..." +source ci/cpu/upload.sh -gpuci_logger "Upload pypi pkg" -source ci/cpu/upload-pypi.sh \ No newline at end of file +gpuci_logger "Upload pypi pkg..." +source ci/cpu/upload-pypi.sh diff --git a/ci/cpu/upload-anaconda.sh b/ci/cpu/upload-anaconda.sh deleted file mode 100755 index 361af0490..000000000 --- a/ci/cpu/upload-anaconda.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh - -set -e - -export UPLOADFILE=`gpuci_conda_retry build conda/recipes/dask-cuda --python=$PYTHON --output` -CUDA_REL=${CUDA_VERSION%.*} - - -LABEL_OPTION="--label main" -echo "LABEL_OPTION=${LABEL_OPTION}" - -# Restrict uploads to master branch -if [ ${BUILD_MODE} != "branch" ]; then - echo "Skipping upload" - return 0 -fi - -if [ -z "$MY_UPLOAD_KEY" ]; then - echo "No upload key" - return 0 -fi - -echo "Upload" -echo ${UPLOADFILE} -anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${UPLOADFILE} diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh new file mode 100755 index 000000000..9fe71454e --- /dev/null +++ b/ci/cpu/upload.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh + +set -e + +# Setup 'gpuci_retry' for upload retries (results in 4 total attempts) +export GPUCI_RETRY_MAX=3 +export GPUCI_RETRY_SLEEP=30 + +# Set default label options if they are not defined elsewhere +export LABEL_OPTION=${LABEL_OPTION:-"--label main"} + +# Skip uploads unless BUILD_MODE == "branch" +if [ ${BUILD_MODE} != "branch" ]; then + echo "Skipping upload" + return 0 +fi + +# Skip uploads if there is no upload key +if [ -z "$MY_UPLOAD_KEY" ]; then + echo "No upload key" + return 0 +fi + +################################################################################ +# SETUP - Get conda file output locations +################################################################################ + +gpuci_logger "Get conda file output locations" +export DASK-CUDA_FILE=`conda build conda/recipes/dask-cuda --python=$PYTHON --output` + +################################################################################ +# UPLOAD - Conda packages +################################################################################ + +gpuci_logger "Starting conda uploads" + +if [ "$UPLOAD_DASK-CUDA" == "1" ]; then + test -e ${DASK-CUDA_FILE} + echo "Upload Dask-cuda" + echo ${DASK-CUDA_FILE} + gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK-CUDA_FILE} +fi From 76ad24102c495179b88b8fcb9714fb0076d08435 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 08:04:03 -0800 Subject: [PATCH 3/6] fixed missing gpuci_conda_retry --- ci/cpu/build.sh | 5 +++++ ci/cpu/upload.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 674b1e59a..cc752c3dd 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -5,6 +5,11 @@ ################################################################################ set -e +# Logger function for build status output +function gpuci_logger() { + echo -e "\n>>>> $@\n" +} + # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 9fe71454e..1f33c1797 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -40,5 +40,5 @@ if [ "$UPLOAD_DASK-CUDA" == "1" ]; then test -e ${DASK-CUDA_FILE} echo "Upload Dask-cuda" echo ${DASK-CUDA_FILE} - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK-CUDA_FILE} + gpuci_conda_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK-CUDA_FILE} fi From 020fccd13a5dc91eb7d2ac00bb976f4416911161 Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 08:31:55 -0800 Subject: [PATCH 4/6] removed gpuci_conda_retry --- ci/cpu/upload.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 1f33c1797..0a7f74650 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -28,7 +28,7 @@ fi ################################################################################ gpuci_logger "Get conda file output locations" -export DASK-CUDA_FILE=`conda build conda/recipes/dask-cuda --python=$PYTHON --output` +export DASKCUDA_FILE=`conda build conda/recipes/dask-cuda --python=$PYTHON --output` ################################################################################ # UPLOAD - Conda packages @@ -36,9 +36,9 @@ export DASK-CUDA_FILE=`conda build conda/recipes/dask-cuda --python=$PYTHON --ou gpuci_logger "Starting conda uploads" -if [ "$UPLOAD_DASK-CUDA" == "1" ]; then - test -e ${DASK-CUDA_FILE} +if [ "$UPLOAD_DASKCUDA" == "1" ]; then + test -e ${DASKCUDA_FILE} echo "Upload Dask-cuda" - echo ${DASK-CUDA_FILE} - gpuci_conda_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK-CUDA_FILE} + echo ${DASKCUDA_FILE} + gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASKCUDA_FILE} fi From 6c63d80df2e0a708edb2f60f3405a0d04245e8da Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Fri, 13 Nov 2020 10:30:08 -0800 Subject: [PATCH 5/6] removed gpuci_logger and replaced parallel_level --- ci/cpu/build.sh | 5 ----- ci/gpu/build.sh | 7 +------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index cc752c3dd..674b1e59a 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -5,11 +5,6 @@ ################################################################################ set -e -# Logger function for build status output -function gpuci_logger() { - echo -e "\n>>>> $@\n" -} - # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 209c6a80d..f45a9e55c 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -7,11 +7,6 @@ set -e NUMARGS=$# ARGS=$* -# Logger function for build status output -function gpuci_logger() { - echo -e "\n>>>> $@\n" -} - # Arg parsing function function hasArg { (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") @@ -19,7 +14,7 @@ function hasArg { # Set path and build parallel level export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=-4 +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} export CUDA_REL=${CUDA_VERSION%.*} export CUDA_REL2=${CUDA//./} From d83fd62210f6e21da8acd413f140da4feb3c60bc Mon Sep 17 00:00:00 2001 From: Mark Sadang Date: Wed, 2 Dec 2020 13:03:13 -0800 Subject: [PATCH 6/6] readded dask pip install --- ci/cpu/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 674b1e59a..02506be94 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -51,6 +51,8 @@ conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False +pip install git+https://github.com/dask/distributed.git@master + ################################################################################ # BUILD - Package builds ################################################################################