Gpuciscripts clean and update

CHANGELOG.md

@@ -9,6 +9,7 @@
 - PR #263 Pin cmake policies to cmake 3.17 version
 - PR #272 CLX Streamz generic structure
 - PR #283 Remove CLX subword tokenizer
+- PR #285 Updates to gpuci scripts
 - PR #286 Update to PyTorch 1.7 for CUDA 11 support
 - PR #287 Dockerfile updates
 

ci/checks/changelog.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# Copyright (c) 2020, NVIDIA CORPORATION.
 #########################
 # clx CHANGELOG Tester #
 #########################

ci/checks/style.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
-# Copyright (c) 2019, NVIDIA CORPORATION.
+# Copyright (c) 2020, NVIDIA CORPORATION.
 ################################################################################
 # clx Style Tester
 ################################################################################
 
 # Ignore errors and set path
 set +e
-PATH=/conda/bin:$PATH
+PATH=/opt/conda/bin:$PATH
 
 # Activate common conda env
-source activate gdf
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
 # Run flake8 and get results/return code
 FLAKE=`flake8 --ignore=E501,W605 --exclude="factory.py,perfect_hash.py" python`

ci/cpu/build.sh

@@ -5,13 +5,9 @@
 ################################################################################
 set -e
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
-# Set path
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+# Set path and build parallel level
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
@@ -24,19 +20,30 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
 fi
 
+# Setup 'conda' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 ################################################################################
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Check environment variables"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check compiler versions"
 python --version
-conda list
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
@@ -45,12 +52,15 @@ conda config --set ssl_verify False
 # BUILD - Conda package build
 ################################################################################
 
-logger "Build conda pkg for clx..."
-source ci/cpu/clx/build_clx.sh
+gpuci_logger "Build conda pkg for libclx"
+gpuci_conda_retry build conda/recipes/libclx
+
+gpuci_logger "Build conda pkg for clx"
+gpuci_conda_retry build -c pytorch -c nvidia -c conda-forge -c defaults conda/recipes/clx --python=$PYTHON
 
 ################################################################################
 # UPLOAD - Conda package
 ################################################################################
 
-logger "Upload clx conda pkg..."
-source ci/cpu/clx/upload-anaconda.sh
+gpuci_logger "Upload conda pkgs"
+source ci/cpu/upload.sh

ci/cpu/upload.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+
+export CLX_FILE=`conda build conda/recipes/clx --python=$PYTHON --output`
+export LIBCLX_FILE=`conda build conda/recipes/libclx --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+if [ "$UPLOAD_CLX" == "1" ]; then
+  test -e ${CLX_FILE}
+  echo "Upload libcudf"
+  echo ${CLX_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CLX_FILE}
+fi
+
+if [ "$UPLOAD_LIBCLX" == "1" ]; then
+  test -e ${LIBCLX_FILE}
+  echo "Upload libcudf"
+  echo ${LIBCLX_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCLX_FILE}
+fi

ci/docs/build.sh

@@ -10,7 +10,7 @@ if [ -z "$PROJECT_WORKSPACE" ]; then
     exit 1
 fi
 
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export HOME=$WORKSPACE
 export DOCS_WORKSPACE=$WORKSPACE/docs
 export NIGHTLY_VERSION=${NIGHTLY_OVERRIDE:=$(echo $BRANCH_VERSION | awk -F. '{ print $2 }')}
@@ -23,29 +23,34 @@ cd $PROJECT_WORKSPACE
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
-conda install --freeze-installed -c rapidsai-nightly -c rapidsai -c nvidia -c pytorch -c conda-forge \
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+gpuci_conda_retry install --freeze-installed -c rapidsai-nightly -c rapidsai -c nvidia -c pytorch -c conda-forge \
     pytorch torchvision requests yaml python-confluent-kafka python-whois markdown beautifulsoup4 jq
 
 pip install mockito
 pip install cupy-cuda${CUDA_SHORT}
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
-conda list
+$CC --version
+$CXX --version
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 #clx source build
 ${PROJECT_WORKSPACE}/build.sh clx
 
 #clx Sphinx Build
-logger "Build clx docs..."
+gpuci_logger "Build clx docs"
 cd ${PROJECT_WORKSPACE}/docs
 make html
 

ci/gpu/build.sh

@@ -3,18 +3,13 @@ set -e
 NUMARGS=$#
 ARGS=$*
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Arg parsing function
 function hasArg {
     (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export CUDA_REL=${CUDA_VERSION%.*}
 export CUDA_SHORT=${CUDA_REL//./}
 
@@ -30,20 +25,21 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Get env"
 env
 
-logger "Activate conda env..."
-source activate rapids
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
-logger "conda install required packages"
-conda install -y -c pytorch -c gwerbin \
+gpuci_logger "conda install required packages"
+gpuci_conda_retry install -y -c pytorch -c gwerbin \
     "rapids-build-env=$MINOR_VERSION.*" \
     "rapids-notebook-env=$MINOR_VERSION.*" \
     "cugraph=${MINOR_VERSION}" \
@@ -59,19 +55,23 @@ conda install -y -c pytorch -c gwerbin \
     "matplotlib" \
     "faker"
 
-logger "pip install git+https://github.com/rapidsai/cudatashader.git"
+gpuci_logger "pip install git+https://github.com/rapidsai/cudatashader.git"
 pip install "git+https://github.com/rapidsai/cudatashader.git"
-logger "pip install mockito"
+gpuci_logger "pip install mockito"
 pip install mockito
 pip install wget
 pip install faker
 
-conda list
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 ################################################################################
 # BUILD - Build clx from source
 ################################################################################
 
+gpuci_logger "Build libclx"
+$WORKSPACE/build.sh clean libclx
 logger "Build clx..."
 $WORKSPACE/build.sh clean clx
 
@@ -83,7 +83,7 @@ EXITCODE=0
 trap "EXITCODE=1" ERR
 
 if hasArg --skip-tests; then
-    logger "Skipping Tests..."
+    gpuci_logger "Skipping Tests"
 else
     cd ${WORKSPACE}/python
     py.test --ignore=ci --cache-clear --junitxml=${WORKSPACE}/junit-clx.xml -v

ci/integration_tests/Dockerfile.test

@@ -13,4 +13,4 @@ RUN source activate rapids \
     && python setup.py install
 
 WORKDIR /clx
-CMD source activate rapids && python run_integration_test.py
+CMD . /opt/conda/etc/profile.d/conda.sh && conda activate rapids && python run_integration_test.py

ci/local/README.md

@@ -30,7 +30,8 @@ For a full list of available gpuCI docker images, visit our [DockerHub](https://
 Style Check:
 ```bash
 $ bash ci/local/build.sh -r ~/rapids/clx -s
-$ source activate gdf    #Activate gpuCI conda environment
+$ . /opt/conda/etc/profile.d/conda.sh   #Activate gpuCI conda environment
+$ conda activate rapids   
 $ cd rapids
 $ flake8 python
 ```