From 750d454239e35f1f43f9e18c26cab54fdb11cec7 Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Tue, 28 Aug 2018 22:59:22 +0200 Subject: [PATCH] [MXNET-908] Enable minimal OSX Travis build --- .travis.yml | 107 ++------- .../travis/install.sh | 16 +- tests/travis/is_core_changed.sh | 60 ----- tests/travis/r_vignettes.R | 21 -- tests/travis/run_test.sh | 210 ------------------ tests/travis/setup.sh | 59 ----- 6 files changed, 25 insertions(+), 448 deletions(-) rename tests/travis/travis_after_failure.sh => ci/travis/install.sh (70%) mode change 100755 => 100644 delete mode 100755 tests/travis/is_core_changed.sh delete mode 100644 tests/travis/r_vignettes.R delete mode 100755 tests/travis/run_test.sh delete mode 100755 tests/travis/setup.sh diff --git a/.travis.yml b/.travis.yml index ca5d03b5008d..f61bd86673dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,108 +1,31 @@ -sudo: false +sudo: true language: cpp +cache: ccache + os: - # - linux - osx -osx_image: xcode8 - -env: - # code analysis - # - TASK=lint - # build mxnet.so with CUDA - # - TASK=build - # run tests/cpp - - TASK=cpp_test - # run tests/python - - TASK=python_test - - TASK=r_test - # - TASK=julia JULIA_VER=0.4 - # - TASK=scala_test - - # TODO, R test, distributed test, clang, more g++ versions +osx_image: xcode9.4 matrix: include: - - # os: linux - # dist: trusty - # env: TASK=perl_test - os: osx - ## sudo is required because - ## prexexisting packages conflict - ## with new ones. - ## would be nice to have macports - ## on travis osx, it has all needed perl packages - sudo: required - env: TASK=perl_test -# env: TASK=julia JULIA_VER=0.4 -# - os: linux -# env: TASK=build -# - os: linux -# env: TASK=cpp_test -# - os: linux -# env: TASK=python_test -# - os: linux -# env: TASK=r_test -# - os: linux -# env: TASK=scala_test - -# dependent apt packages -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - doxygen - - wget - - git - - libcurl4-openssl-dev - - unzip - - libatlas-dev - - libopencv-dev - - gcc-4.8 - - g++-4.8 - - python-numpy - - python-nose - - python3-numpy - - python3-dev - - python3-nose - - python-h5py - - python3-h5py - - graphviz - - libmouse-perl - - pdl - - cpanminus - - swig - - libgraphviz-perl before_install: - - export NVCC_PREFIX=${HOME} - - source dmlc-core/scripts/travis/travis_setup_env.sh - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - - export MAVEN_SKIP_RC=true - - export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" install: - - source tests/travis/setup.sh - + - brew install ccache + - export PATH="/usr/local/opt/ccache/libexec:$PATH" + - source ci/travis/install.sh + +# We build with 2 concurrent jobs to match the number of cores present on MacOS virutal machines. +# nproc does not report the correct number of cores reliably in Travis, so using nproc is not +# recommended. +# https://docs.travis-ci.com/user/reference/overview/ script: - - tests/travis/run_test.sh - -cache: - directories: - - ${HOME}/.cache/usr - -before_cache: - - dmlc-core/scripts/travis/travis_before_cache.sh - -after_failure: - - tests/travis/travis_after_failure.sh - -notifications: -# Emails are sent to the committer's git-configured email address by default, - email: - on_success: change - on_failure: always - #slack: dmlc:NmroCzntCiWOuxUZpii40USd + - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + - mv make/osx.mk config.mk + - make -j 2 diff --git a/tests/travis/travis_after_failure.sh b/ci/travis/install.sh old mode 100755 new mode 100644 similarity index 70% rename from tests/travis/travis_after_failure.sh rename to ci/travis/install.sh index 50754c9546cd..d04dda7e87f3 --- a/tests/travis/travis_after_failure.sh +++ b/ci/travis/install.sh @@ -17,10 +17,14 @@ # specific language governing permissions and limitations # under the License. - -if [ ${TASK} == "r_test" ]; then - echo "Print the install log..." - cat mxnet.Rcheck/*.out - echo "Print the check log..." - cat mxnet.Rcheck/*.log +if [ ${TRAVIS_OS_NAME} == "osx" ]; then + brew update + brew install opencv + brew install python3 + brew install fftw + brew install libpng + brew install ImageMagick + brew install swig + python -m pip install --user nose numpy cython scipy requests + python3 -m pip install --user nose numpy cython scipy requests fi diff --git a/tests/travis/is_core_changed.sh b/tests/travis/is_core_changed.sh deleted file mode 100755 index 7b9eb6123847..000000000000 --- a/tests/travis/is_core_changed.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# this is a util script to test whether the "core" of -# mxnet has changed. Please modify the regex patterns here -# to ensure the components are covered if you add new "core" -# components to mxnet - -# temporarily disable this b/c the OS X tests are failing mysteriously -exit 0 - -# DEBUG -echo "Files changed in this PR includes:" -echo "**********************************" -git diff --name-only HEAD^ -echo "**********************************" - -# we ignore examples, and docs -core_patterns=( - '^dmlc-core' - '^matlab' - '^plugin' - '^python' - '^src' - '^tools' - '^R-package' - '^amalgamation' - '^include' - '^mshadow' - '^ps-lite' - '^scala-package' - '^tests' -) - -for pat in ${core_patterns[@]}; do - if git diff --name-only HEAD^ | grep "$pat" - then - exit - fi -done - -echo "I think we are good to skip this travis ci run now" -exit 1 # means nothing has changed diff --git a/tests/travis/r_vignettes.R b/tests/travis/r_vignettes.R deleted file mode 100644 index 1b03b8bba4ec..000000000000 --- a/tests/travis/r_vignettes.R +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -fnames <- list.files("R-package/vignettes/", pattern="*.Rmd") -sapply(fnames, function(x){ - knitr::purl(paste0("R-package/vignettes/", x)) - }) \ No newline at end of file diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh deleted file mode 100755 index fd23f0e82b24..000000000000 --- a/tests/travis/run_test.sh +++ /dev/null @@ -1,210 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -if ! tests/travis/is_core_changed.sh -then - exit 0 -fi - -if [ ${TASK} == "lint" ]; then - make lint || exit -1 - echo "Check documentations of c++ code..." - make doc 2>log.txt - (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt - echo "---------Error Log----------" - cat logclean.txt - echo "----------------------------" - (cat logclean.txt|grep warning) && exit -1 - (cat logclean.txt|grep error) && exit -1 - exit 0 -fi - -cp make/config.mk config.mk - -if [[ ${TRAVIS_OS_NAME} == "osx" ]]; then - echo "USE_BLAS=apple" >> config.mk - echo "USE_OPENMP=0" >> config.mk -else - # use g++-4.8 for linux - if [[ ${CXX} == "g++" ]]; then - export CXX=g++-4.8 - fi - echo "USE_BLAS=blas" >> config.mk -fi -echo "CXX=${CXX}" >>config.mk -echo "USE_PROFILER=1" >> config.mk - -if [ ${TASK} == "build" ]; then - if [ ${TRAVIS_OS_NAME} == "linux" ]; then - echo "USE_CUDA=1" >> config.mk - ./dmlc-core/scripts/setup_nvcc.sh $NVCC_PREFIX - fi - make all - exit $? -fi - -if [ ${TASK} == "cpp_test" ]; then - make -f dmlc-core/scripts/packages.mk gtest - echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk - make test || exit -1 - export MXNET_ENGINE_INFO=true - ./build/tests/cpp/mxnet_test - exit 0 -fi - -if [ ${TASK} == "r_test" ]; then - make all || exit -1 - # use cached dir for storing data - rm -rf ${PWD}/data - mkdir -p ${CACHE_PREFIX}/data - ln -s ${CACHE_PREFIX}/data ${PWD}/data - - set -e - export _R_CHECK_TIMINGS_=0 - - if [[ ${TRAVIS_OS_NAME} == "osx" ]]; then - wget https://cran.rstudio.com/bin/macosx/R-latest.pkg -O /tmp/R-latest.pkg - sudo installer -pkg "/tmp/R-latest.pkg" -target / - Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" - fi - - cd R-package - Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" - cd .. - - make rpkg -# R CMD check --no-examples --no-manual --no-vignettes --no-build-vignettes mxnet_*.tar.gz - R CMD INSTALL mxnet_*.tar.gz - - Rscript tests/travis/r_vignettes.R - - wget http://data.mxnet.io/mxnet/data/Inception.zip - unzip Inception.zip && rm -rf Inception.zip - wget http://data.mxnet.io/mxnet/data/mnist.zip - unzip mnist.zip && rm -rf mnist.zip - - cat CallbackFunctionTutorial.R \ - fiveMinutesNeuralNetwork.R \ - mnistCompetition.R \ - ndarrayAndSymbolTutorial.R > r_test.R - - Rscript r_test.R || exit -1 - - exit 0 -fi - -if [ ${TASK} == "python_test" ]; then - make all || exit -1 - # use cached dir for storing data - rm -rf ${PWD}/data - mkdir -p ${PWD}/data - - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - python -m nose -v tests/python/unittest || exit -1 - python3 -m nose -v tests/python/unittest || exit -1 - # make cython3 - # cython tests - # export MXNET_ENFORCE_CYTHON=1 - # python3 -m nose tests/python/unittest || exit -1 - python3 -m nose -v tests/python/train || exit -1 - python -m nose -v tests/python/doctest || exit -1 - python3 -m nose -v tests/python/doctest || exit -1 - else - nosetests -v tests/python/unittest || exit -1 - nosetests3 -v tests/python/unittest || exit -1 - nosetests3 -v tests/python/train || exit -1 - nosetests -v tests/python/doctest || exit -1 - nosetests3 -v tests/python/doctest || exit -1 - fi - exit 0 -fi - -if [ ${TASK} == "julia" ]; then - make all || exit -1 - # use cached dir for storing data - rm -rf ${PWD}/data - mkdir -p ${PWD}/data - - export MXNET_HOME="${PWD}" - julia -e 'Pkg.clone("MXNet"); Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1 - exit 0 -fi - -if [ ${TASK} == "scala_test" ]; then - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - LIB_GOMP_PATH=`find /usr/local/lib -name libgomp.dylib | grep -v i386 | head -n1` - ln -sf $LIB_GOMP_PATH /usr/local/lib/libgomp.dylib - fi - make all || exit -1 - # use cached dir for storing data - rm -rf ${PWD}/data - mkdir -p ${PWD}/data - - export JAVA_HOME=$(/usr/libexec/java_home) - - make scalapkg || exit -1 - make scalatest || exit -1 - - exit 0 -fi - -if [ ${TASK} == "perl_test" ]; then - make all || exit -1 - - # use cached dir for storing data - MXNET_HOME=${PWD} - rm -rf ${MXNET_HOME}/perl-package/AI-MXNet/data - mkdir -p ${CACHE_PREFIX}/data - ln -s ${CACHE_PREFIX}/data ${MXNET_HOME}/perl-package/AI-MXNet/data - - export LD_LIBRARY_PATH=${MXNET_HOME}/lib - export PERL5LIB=${HOME}/perl5/lib/perl5 - - cd ${MXNET_HOME}/perl-package/AI-MXNetCAPI/ - perl Makefile.PL INSTALL_BASE=${HOME}/perl5 - make || exit -1 - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - install_name_tool -change lib/libmxnet.so \ - ${MXNET_HOME}/lib/libmxnet.so \ - blib/arch/auto/AI/MXNetCAPI/MXNetCAPI.bundle - fi - make install || exit -1 - - cd ${MXNET_HOME}/perl-package/AI-NNVMCAPI/ - perl Makefile.PL INSTALL_BASE=${HOME}/perl5 - make || exit -1 - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - install_name_tool -change lib/libmxnet.so \ - ${MXNET_HOME}/lib/libmxnet.so \ - blib/arch/auto/AI/NNVMCAPI/NNVMCAPI.bundle - fi - make install || exit -1 - - cd ${MXNET_HOME}/perl-package/AI-MXNet/ - perl Makefile.PL - make test || exit -1 - exit 0 -fi - -if [ ${TASK} == "cpp_package_test" ]; then - MXNET_HOME=${PWD} - make travis -C ${MXNET_HOME}/cpp-package/example - exit 0 -fi diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh deleted file mode 100755 index eec6c23d7158..000000000000 --- a/tests/travis/setup.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -if ! tests/travis/is_core_changed.sh -then - exit 0 -fi - -if [ ${TRAVIS_OS_NAME} == "osx" ]; then - brew update - brew tap homebrew/science - brew install opencv - brew install python3 - brew install fftw - brew install libpng - brew install ImageMagick - brew install swig - if [ ${TASK} == "python_test" ]; then - python -m pip install --user nose numpy cython scipy - python3 -m pip install --user nose numpy cython scipy - fi -fi - -if [ ${TASK} == "lint" ]; then - pip install --user cpplint 'pylint==1.4.4' 'astroid==1.3.6' -fi - -if [ ${TASK} == "julia" ]; then - mkdir -p ~/julia - curl -s -L --retry 7 "https://s3.amazonaws.com/julialang/bin/linux/x64/${JULIA_VER}/julia-${JULIA_VER}-latest-linux-x86_64.tar.gz" | tar -C ~/julia -x -z --strip-components=1 -f - - export PATH="${PATH}:${HOME}/julia/bin" - julia -e 'versioninfo()' -fi - -if [ ${TASK} == "perl_test" ]; then - if [ ${TRAVIS_OS_NAME} == "linux" ]; then - cpanm -q -L "${HOME}/perl5" Function::Parameters Hash::Ordered PDL::CCS - else - sudo sh -c 'curl -L https://cpanmin.us | perl - App::cpanminus' - sudo cpanm -q -n PDL Mouse Function::Parameters Hash::Ordered PDL::CCS - fi -fi