From 58955e56d3799b43b922afeccca241b0ebfeb1d5 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 17 May 2016 20:33:23 -0700 Subject: [PATCH 1/2] Draft of automated conda builds for libarrow, pyarrow. Remove unneeded thrift-cpp build dependency --- .travis.yml | 27 +++++++++++++++++++- ci/travis_conda_build.sh | 51 ++++++++++++++++++++++++++++++++++++++ cpp/conda.recipe/build.sh | 15 ++++++++++- cpp/conda.recipe/meta.yaml | 5 ++-- 4 files changed, 93 insertions(+), 5 deletions(-) create mode 100755 ci/travis_conda_build.sh diff --git a/.travis.yml b/.travis.yml index a0138a79598..9998f8f972e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: precise +dist: precise addons: apt: sources: @@ -18,6 +18,9 @@ addons: - valgrind matrix: + fast_finish: true + allow_failures: + - env: ARROW_TEST_GROUP=packaging include: - compiler: gcc language: cpp @@ -39,6 +42,24 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh + - compiler: gcc + env: ARROW_TEST_GROUP=packaging + os: linux + before_script: + - export CC="gcc-4.9" + - export CXX="g++-4.9" + script: + - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh + - os: osx + env: ARROW_TEST_GROUP=packaging + language: objective-c + osx_image: xcode6.4 + compiler: clang + addons: + before_script: + before_install: + script: + - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh before_install: - ulimit -c unlimited -S @@ -51,3 +72,7 @@ after_script: after_failure: - COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1) - if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi + +env: + global: + - secure: "nb1X9Sicq7fqQ4el1R3ugr+3CQdWQhQ+PbaDIya9xsKJ6HfMcitj1M/r5mLg8zgphMt4/lcknsnOuwo8beZpVJLvgAy7WFsrPBrCAeWe2ETpssry1nJoN29VrHfazkJ21LzpxwoYmOvGEj8xHaNZYJfH40DqGFIHVA/TtB825tZpQmNv12Ewtalt6+j8emrPfEUs/v6VjlQ3/1Kq6XnbaUD/KaKytlxhRnYSGYnxZ12sSsRl9CIC9y9jccLkat9CGFgH4dVBXTWVa5LJ2hxVpajj5k/khKPfgU1B9GWKdFhGXJu6ju85MA4XHDN0crE2jBw1fPeI1MqzXRodfYAYNlUm6+h/kEnzGH5Tu2mRw2Oc8NpKfNwkOuUZgOcD0Wai/mC0VdlkKukxUOMI7rCgSBJqgmp3OiHsQlu+Pm9lbwsef5/8icVX68/0ISnI/i9L04sgHExFgVY/PfJKGa/N25MWmBVXjH9aT9tgQEdhv3hzplpcNgdPmpr3VQXdWlYLMeBGxInHDYnNJCkdEjbww/oDhVBDB+tMnbv05qmbIdeCtZz1Z92/CabZlqpa/y+ovdhp9qUOoWIWTfG5VbhX1BxsA308ab2xDPC/bXmmm9GgTN9vsT7kSO4YZ+vV+srfXnmSquo5zKSpzsNI3dVA98zNBmxo9hr3Ugpucnm3Kfs=" diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh new file mode 100755 index 00000000000..64720177163 --- /dev/null +++ b/ci/travis_conda_build.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e + +if [ $TRAVIS_OS_NAME == "linux" ]; then + MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh" +else + MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh" +fi + +wget -O miniconda.sh $MINICONDA_URL +MINICONDA=$TRAVIS_BUILD_DIR/miniconda +bash miniconda.sh -b -p $MINICONDA +export PATH="$MINICONDA/bin:$PATH" +conda update -y -q conda +conda info -a + +conda config --set show_channel_urls yes +conda config --add channels conda-forge +conda config --add channels apache + +conda install --yes conda-build jinja2 anaconda-client + +# faster builds, please +conda install -y nomkl + +# Build libarrow + +cd $TRAVIS_BUILD_DIR/cpp + +conda build conda.recipe --channel apache/channel/dev +CONDA_PACKAGE=`conda build --output conda.recipe | grep bz2` +anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; + +# if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then +# anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; +# fi + +# Build pyarrow + +cd $TRAVIS_BUILD_DIR/python + +build_for_python_version() { + PY_VERSION=$1 + conda build conda.recipe --python $PY_VERSION --channel apache/channel/dev + CONDA_PACKAGE=`conda build --python $PY_VERSION --output conda.recipe | grep bz2` + anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev +} + +build_for_python_version 2.7 +build_for_python_version 3.5 diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh index ac1f9c89cc9..b10dd03349b 100644 --- a/cpp/conda.recipe/build.sh +++ b/cpp/conda.recipe/build.sh @@ -9,6 +9,19 @@ cd $RECIPE_DIR export FLATBUFFERS_HOME=$PREFIX export PARQUET_HOME=$PREFIX +if [ "$(uname)" == "Darwin" ]; then + # C++11 finagling for Mac OSX + export CC=clang + export CXX=clang++ + export MACOSX_VERSION_MIN="10.7" + CXXFLAGS="${CXXFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}" + CXXFLAGS="${CXXFLAGS} -stdlib=libc++ -std=c++11" + export LDFLAGS="${LDFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}" + export LDFLAGS="${LDFLAGS} -stdlib=libc++ -std=c++11" + export LINKFLAGS="${LDFLAGS}" + export MACOSX_DEPLOYMENT_TARGET=10.7 +fi + cd .. rm -rf conda-build @@ -33,7 +46,7 @@ elif [ `uname` == Darwin ]; then fi cmake \ - -DCMAKE_BUILD_TYPE=debug \ + -DCMAKE_BUILD_TYPE=release \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \ -DARROW_IPC=on \ diff --git a/cpp/conda.recipe/meta.yaml b/cpp/conda.recipe/meta.yaml index 2e834d5cbf8..75f3a8ba3d9 100644 --- a/cpp/conda.recipe/meta.yaml +++ b/cpp/conda.recipe/meta.yaml @@ -15,15 +15,14 @@ requirements: - cmake - flatbuffers - parquet-cpp - - thrift-cpp run: - parquet-cpp test: commands: - - test -f $PREFIX/lib/libarrow.so - - test -f $PREFIX/lib/libarrow_parquet.so + - test -f $PREFIX/lib/libarrow.so # [linux] + - test -f $PREFIX/lib/libarrow_parquet.so # [linux] - test -f $PREFIX/include/arrow/api.h about: From afd0582832af85ffc1b4e8e669274138f9239e76 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 17 May 2016 21:17:34 -0700 Subject: [PATCH 2/2] Change encrypted token to apache/arrow, only upload on commits to master --- .travis.yml | 2 +- ci/travis_conda_build.sh | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9998f8f972e..646f80fee7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,4 +75,4 @@ after_failure: env: global: - - secure: "nb1X9Sicq7fqQ4el1R3ugr+3CQdWQhQ+PbaDIya9xsKJ6HfMcitj1M/r5mLg8zgphMt4/lcknsnOuwo8beZpVJLvgAy7WFsrPBrCAeWe2ETpssry1nJoN29VrHfazkJ21LzpxwoYmOvGEj8xHaNZYJfH40DqGFIHVA/TtB825tZpQmNv12Ewtalt6+j8emrPfEUs/v6VjlQ3/1Kq6XnbaUD/KaKytlxhRnYSGYnxZ12sSsRl9CIC9y9jccLkat9CGFgH4dVBXTWVa5LJ2hxVpajj5k/khKPfgU1B9GWKdFhGXJu6ju85MA4XHDN0crE2jBw1fPeI1MqzXRodfYAYNlUm6+h/kEnzGH5Tu2mRw2Oc8NpKfNwkOuUZgOcD0Wai/mC0VdlkKukxUOMI7rCgSBJqgmp3OiHsQlu+Pm9lbwsef5/8icVX68/0ISnI/i9L04sgHExFgVY/PfJKGa/N25MWmBVXjH9aT9tgQEdhv3hzplpcNgdPmpr3VQXdWlYLMeBGxInHDYnNJCkdEjbww/oDhVBDB+tMnbv05qmbIdeCtZz1Z92/CabZlqpa/y+ovdhp9qUOoWIWTfG5VbhX1BxsA308ab2xDPC/bXmmm9GgTN9vsT7kSO4YZ+vV+srfXnmSquo5zKSpzsNI3dVA98zNBmxo9hr3Ugpucnm3Kfs=" + - secure: "GcrPtsKUCgNY7HKYjWlHQo8SiFrShDvdZSU8t1m1FJrE+UfK0Dgh9zXmAausM8GmhqSwkF0q4UbLQf2uCnSITWKeEPAL8Mo9eu4ib+ikJx/b3Sk81frgW5ADoHfW1Eyqd8xJNIMwMegJOtRLSDqiXh1CvMlKnY8PyTOGM2DgN9ona/v6p9OFH9Qs0JhBRVXAn0S4ztjumck8E56+01hqRfxbZ88pTfpKghBxYp9PJaMjtGdomjVWlqPaWaWJj+KptT8inV9NK+TVYKx0dXWD+S1Vgr1PytQnLdILOYV23gsOBYqn33ByF/yADl4m3hUjU/qeT0Fi7aWxmVpj+oTJISOSH5N8nIsuNH8mQk2ZzzXHfV7btFvP+cOPRczadoKkT6D6cHA8nQ7b0dphC6bl6SAeSfc/cbhRT+fYnIjg8jFXC8jlyWBr7LR6GXVpc0bND7i300ITo0FuRJhy2OxqPtGo3dKLE7eAcv78tuO0OYJ/kol1PEqFdFkbYbNVbg/cFpbGqiCXDsOtPDbAGBv69YnXdVowSxxs8cRGjSkDydv6ZSytb/Zd4lH/KAomcFNk8adx12O1Lk4sbmVav1cGig5P6OcQKS0jC5IiRb4THcQzVzAkXXbaafKm5sru/NoYxhzmkyhkOc11nTYHKVng+XKWzLCNn7pTTSLitp5+xa4=" diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh index 64720177163..afa531dbd6b 100755 --- a/ci/travis_conda_build.sh +++ b/ci/travis_conda_build.sh @@ -30,21 +30,23 @@ cd $TRAVIS_BUILD_DIR/cpp conda build conda.recipe --channel apache/channel/dev CONDA_PACKAGE=`conda build --output conda.recipe | grep bz2` -anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; -# if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then -# anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; -# fi +if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then + anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; +fi # Build pyarrow cd $TRAVIS_BUILD_DIR/python build_for_python_version() { - PY_VERSION=$1 - conda build conda.recipe --python $PY_VERSION --channel apache/channel/dev - CONDA_PACKAGE=`conda build --python $PY_VERSION --output conda.recipe | grep bz2` - anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev + PY_VERSION=$1 + conda build conda.recipe --python $PY_VERSION --channel apache/channel/dev + CONDA_PACKAGE=`conda build --python $PY_VERSION --output conda.recipe | grep bz2` + + if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then + anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev; + fi } build_for_python_version 2.7