Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 31 additions & 24 deletions ci/cpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,20 @@
######################################
set -e

# Logger function for build status output
function logger() {
echo -e "\n>>>> $@\n"
}

# Set path and build parallel level
export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=4
export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}

# Set home to the job's workspace
export HOME=$WORKSPACE

# Switch to project root; also root of repo checkout
cd $WORKSPACE

# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
export GPUCI_CONDA_RETRY_MAX=1
export GPUCI_CONDA_RETRY_SLEEP=30

# If nightly build, append current YYMMDD to version
if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
export VERSION_SUFFIX=`date +%y%m%d`
Expand All @@ -29,44 +28,52 @@ fi
# SETUP - Check environment
################################################################################

logger "Get env..."
gpuci_logger "Get env"
env

logger "Activate conda env..."
source activate gdf
gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate rapids

logger "Check versions..."
gpuci_logger "Check versions"
python --version
gcc --version
g++ --version
conda list

gpuci_logger "Check conda environment"
conda info
conda config --show-sources
conda list --show-channel-urls

# FIX Added to deal with Anancoda SSL verification issues during conda builds
conda config --set ssl_verify False

################################################################################
# BUILD - Conda package builds (conda deps: librmm <- rmm)
################################################################################

if [[ "$BUILD_LIBRMM" == "1" ]]; then
logger "Build conda pkg for librmm..."
source ci/cpu/librmm/build_librmm.sh
gpuci_logger "Build conda pkg for librmm"
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
conda build conda/recipes/librmm --python=$PYTHON
else
conda build --dirty --no-remove-work-dir conda/recipes/librmm
fi
fi

if [[ "$BUILD_RMM" == "1" ]]; then
logger "Build conda pkg for rmm..."
source ci/cpu/rmm/build_rmm.sh
gpuci_logger "Build conda pkg for rmm"
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
conda build conda/recipes/rmm --python=$PYTHON
else
conda build --dirty --no-remove-work-dir conda/recipes/rmm
fi
fi

################################################################################
# UPLOAD - Conda packages
################################################################################

if [[ "$BUILD_LIBRMM" == "1" ]]; then
logger "Upload conda pkg for librmm..."
source ci/cpu/librmm/upload-anaconda.sh
fi
gpuci_logger "Upload conda packages"
source ci/cpu/upload.sh

if [[ "$BUILD_RMM" == "1" ]]; then
logger "Upload conda pkg for rmm..."
source ci/cpu/rmm/upload-anaconda.sh
fi
9 changes: 0 additions & 9 deletions ci/cpu/librmm/build_librmm.sh

This file was deleted.

31 changes: 0 additions & 31 deletions ci/cpu/librmm/upload-anaconda.sh

This file was deleted.

9 changes: 0 additions & 9 deletions ci/cpu/rmm/build_rmm.sh

This file was deleted.

30 changes: 0 additions & 30 deletions ci/cpu/rmm/upload-anaconda.sh

This file was deleted.

54 changes: 54 additions & 0 deletions ci/cpu/upload.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash
#
# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh

set -e

# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
export GPUCI_RETRY_MAX=3
export GPUCI_RETRY_SLEEP=30

# Set default label options if they are not defined elsewhere
export LABEL_OPTION=${LABEL_OPTION:-"--label main"}

# Skip uploads unless BUILD_MODE == "branch"
if [ ${BUILD_MODE} != "branch" ]; then
echo "Skipping upload"
return 0
fi

# Skip uploads if there is no upload key
if [ -z "$MY_UPLOAD_KEY" ]; then
echo "No upload key"
return 0
fi

################################################################################
# SETUP - Get conda file output locations
################################################################################

gpuci_logger "Get conda file output locations"

export LIBRMM_FILE=`conda build conda/recipes/librmm --output`
export RMM_FILE=`conda build conda/recipes/rmm --python=$PYTHON --output`

################################################################################
# UPLOAD - Conda packages
################################################################################

gpuci_logger "Starting conda uploads"

if [ "$BUILD_LIBRMM" == "1" && "$UPLOAD_LIBRMM" == "1" ]; then
test -e ${LIBRMM_FILE}
echo "Upload librmm"
echo ${LIBRMM_FILE}
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRMM_FILE}
fi

if [ "$BUILD_RMM" == "1" && "$UPLOAD_RMM" == "1" ]; then
test -e ${RMM_FILE}
echo "Upload rmm"
echo ${RMM_FILE}
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${RMM_FILE}
fi

23 changes: 14 additions & 9 deletions ci/docs/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,31 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
export NIGHTLY_VERSION=$(echo $BRANCH_VERSION | awk -F. '{print $2}')
export PROJECTS=(rmm)

logger "Check environment..."
gpuci_logger "Check environment"
env

logger "Check GPU usage..."
gpuci_logger "Check GPU usage"
nvidia-smi

logger "Activate conda env..."
source activate rapids
# TODO: Move installs to docs-build-env meta package
conda install -c anaconda beautifulsoup4 jq
gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate rapids

# TODO: Move installs to docs-build-env meta package
gpuci_conda_retry install -c anaconda beautifulsoup4 jq
Comment thread
dillon-cullinan marked this conversation as resolved.
Resolved

logger "Check versions..."
gpuci_logger "Check versions"
python --version
$CC --version
$CXX --version
conda list

gpuci_logger "Check conda environment"
conda info
conda config --show-sources
conda list --show-channel-urls

# Build Doxygen docs
logger "Build Doxygen docs..."
gpuci_logger "Build Doxygen docs"
cd $PROJECT_WORKSPACE/doxygen
doxygen Doxyfile

Expand Down
54 changes: 27 additions & 27 deletions ci/gpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,14 @@ set -e
NUMARGS=$#
ARGS=$*

# Logger function for build status output
function logger() {
echo -e "\n>>>> $@\n"
}

# Arg parsing function
function hasArg {
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
}

# Set path and build parallel level
export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=4
export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}

# Set home to the job's workspace
export HOME=$WORKSPACE
Expand All @@ -36,49 +31,54 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
# SETUP - Check environment
################################################################################

logger "Get env..."
gpuci_logger "Get env"
env

logger "Activate conda env..."
source activate gdf
gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate rapids

# Install build env
conda install rapids-build-env=${MINOR_VERSION}.*
gpuci_conda_retry install rapids-build-env=${MINOR_VERSION}.*

# https://docs.rapids.ai/maintainers/depmgmt/
# conda remove -f rapids-build-env
# conda install "your-pkg=1.0.0"
# gpuci_conda_retry remove -f rapids-build-env
# gpuci_conda_retry install "your-pkg=1.0.0"

logger "Check versions..."
gpuci_logger "Check versions"
python --version
gcc --version
g++ --version
conda list
$CC --version
$CXX --version

gpuci_logger "Check conda environment"
conda info
conda config --show-sources
conda list --show-channel-urls

if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
################################################################################
# BUILD - Build and install librmm and rmm
################################################################################

logger "Build and install librmm and rmm..."
gpuci_logger "Build and install librmm and rmm"
"$WORKSPACE/build.sh" -v clean librmm rmm

################################################################################
# Test - librmm
################################################################################

if hasArg --skip-tests; then
logger "Skipping Tests..."
gpuci_logger "Skipping Tests"
else
logger "Check GPU usage..."
gpuci_logger "Check GPU usage"
nvidia-smi

logger "Running googletests..."
gpuci_logger "Running googletests"

cd "${WORKSPACE}/build"
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/" make -j${PARALLEL_LEVEL} test

logger "Python py.test for librmm_cffi..."
gpuci_logger "Python py.test for librmm_cffi"
cd $WORKSPACE/python
py.test --cache-clear --basetemp=${WORKSPACE}/rmm-cuda-tmp --junitxml=${WORKSPACE}/test-results/junit-rmm.xml -v --cov-config=.coveragerc --cov=rmm --cov-report=xml:${WORKSPACE}/python/rmm-coverage.xml --cov-report term
fi
Expand All @@ -89,10 +89,10 @@ else
mkdir -p ${TESTRESULTS_DIR}
SUITEERROR=0

logger "Check GPU usage..."
gpuci_logger "Check GPU usage"
nvidia-smi

logger "Running googletests..."
gpuci_logger "Running googletests"
# run gtests
cd $WORKSPACE/ci/artifacts/rmm/cpu/conda_work
for gt in "build/gtests/*" ; do
Expand All @@ -106,14 +106,14 @@ else

cd $WORKSPACE/python

logger "Installing librmm..."
gpuci_logger "Installing librmm"
conda install -c $WORKSPACE/ci/artifacts/rmm/cpu/conda-bld/ librmm
export LIBRMM_BUILD_DIR="$WORKSPACE/ci/artifacts/rmm/cpu/conda_work/build"

logger "Building rmm"
gpuci_logger "Building rmm"
"$WORKSPACE/build.sh" -v rmm

logger "pytest rmm"
gpuci_logger "pytest rmm"
py.test --cache-clear --junitxml=${WORKSPACE}/test-results/junit-rmm.xml -v --cov-config=.coveragerc --cov=rmm --cov-report=xml:${WORKSPACE}/python/rmm-coverage.xml --cov-report term
exitcode=$?
if (( ${exitcode} != 0 )); then
Expand Down
Loading