From 86e7e5bcbeabdac98b06df4d56067ad023878446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 25 Jan 2019 12:52:44 +0100 Subject: [PATCH 01/34] run pyarrow tests in manylinux CI build --- ci/travis_script_manylinux.sh | 18 +++++++++++++++++- dev/tasks/python-wheels/linux-test.sh | 3 +-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 8e8086aa32d..f4ce4e9d091 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -39,4 +39,20 @@ conda activate $CONDA_ENV_DIR pip install -q tensorflow pip install "dist/`ls dist/ | grep cp36`" -python -c "import pyarrow; import tensorflow" + +# # Test optional dependencies and the presence of tensorflow +python -c " +import sys +import pyarrow +import pyarrow.orc +import pyarrow.parquet +import pyarrow.plasma +import tensorflow + +if sys.version_info.major > 2: + import pyarrow.gandiva +" + +# Run pyarrow tests +pip install -r /arrow/python/requirements-test.txt +pytest --pyargs pyarrow diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/linux-test.sh index 45efdb0c91e..fd660dc5754 100755 --- a/dev/tasks/python-wheels/linux-test.sh +++ b/dev/tasks/python-wheels/linux-test.sh @@ -24,7 +24,7 @@ pip install /arrow/python/manylinux1/dist/*.whl python --version # Test optional dependencies -command=" +python -c " import sys import pyarrow import pyarrow.orc @@ -34,7 +34,6 @@ import pyarrow.plasma if sys.version_info.major > 2: import pyarrow.gandiva " -python -c "$command" # Run pyarrow tests pip install -r /arrow/python/requirements-test.txt From 97fe94b23bd224da3b511a4e23f444b263b140d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 25 Jan 2019 12:59:31 +0100 Subject: [PATCH 02/34] fix requirements-test.txt path --- ci/travis_script_manylinux.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index f4ce4e9d091..fd5ad443cce 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -39,6 +39,7 @@ conda activate $CONDA_ENV_DIR pip install -q tensorflow pip install "dist/`ls dist/ | grep cp36`" +popd # # Test optional dependencies and the presence of tensorflow python -c " @@ -54,5 +55,5 @@ if sys.version_info.major > 2: " # Run pyarrow tests -pip install -r /arrow/python/requirements-test.txt +pip install -r python/requirements-test.txt pytest --pyargs pyarrow From 00d98e0db4ff2eeef907507b0f9fd02893caf4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 28 Jan 2019 16:09:40 +0100 Subject: [PATCH 03/34] try to bundle bytecode files --- .travis.yml | 2 ++ ci/travis_script_manylinux.sh | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60c44462f6c..7bfe10ac059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -225,6 +225,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" language: cpp + env: + - PYTHON_VERSIONS=2.7,16 3.6,16 before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index fd5ad443cce..239f0a65c97 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -21,14 +21,18 @@ set -ex pushd python/manylinux1 -docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 /io/build_arrow.sh +docker run --shm-size=2g --rm \ + -e PYARROW_PARALLEL=3 \ + -v $PWD:/io \ + -v $PWD/../../:/arrow \ + quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 \ + /io/build_arrow.sh # Testing for https://issues.apache.org/jira/browse/ARROW-2657 # These tests cannot be run inside of the docker container, since TensorFlow # does not run on manylinux1 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh PYTHON_VERSION=3.6 From dbf5b1cf5b48dad30479e3e91457bb1722e64735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 12:54:13 +0100 Subject: [PATCH 04/34] embed precompiled bitcode as char array; load precompiled IR from string --- cpp/src/gandiva/CMakeLists.txt | 22 +++++++--------- cpp/src/gandiva/configuration.cc | 10 +++---- cpp/src/gandiva/configuration.h | 26 +++++++++---------- cpp/src/gandiva/engine.cc | 13 +++++----- cpp/src/gandiva/engine.h | 4 +-- cpp/src/gandiva/precompiled/CMakeLists.txt | 25 +++++++++++++++--- ...e_path.cc.in => precompiled_bitcode.cc.in} | 5 ++-- cpp/src/gandiva/projector_cache_key.h | 2 +- cpp/src/gandiva/tests/projector_test.cc | 22 ++++++++-------- cpp/src/gandiva/tests/test_util.h | 3 +-- 10 files changed, 73 insertions(+), 59 deletions(-) rename cpp/src/gandiva/{bc_file_path.cc.in => precompiled_bitcode.cc.in} (80%) diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 068a21ee1d6..b0dc3bf1a83 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -27,17 +27,14 @@ add_dependencies(gandiva-all gandiva gandiva-tests gandiva-benchmarks) find_package(LLVM) -# Set the path where the byte-code files will be installed. -set(GANDIVA_BC_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/gandiva) - -set(GANDIVA_BC_FILE_NAME irhelpers.bc) -set(GANDIVA_BC_INSTALL_PATH "${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME}") -set(GANDIVA_BC_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${GANDIVA_BC_FILE_NAME}") -install(FILES ${GANDIVA_BC_OUTPUT_PATH} DESTINATION ${GANDIVA_BC_INSTALL_DIR}) - -set(BC_FILE_PATH_CC "${CMAKE_CURRENT_BINARY_DIR}/bc_file_path.cc") -configure_file(bc_file_path.cc.in ${BC_FILE_PATH_CC}) -add_definitions(-DGANDIVA_BYTE_COMPILE_FILE_PATH="${GANDIVA_BC_OUTPUT_PATH}") +# Set the path where the bitcode file generated, see precompiled/CMakeLists.txt +set(GANDIVA_PRECOMPILED_BC_PATH "${CMAKE_CURRENT_BINARY_DIR}/irhelpers.bc") +set(GANDIVA_PRECOMPILED_CC_PATH "${CMAKE_CURRENT_BINARY_DIR}/precompiled_bitcode.cc") +set(GANDIVA_PRECOMPILED_CC_IN_PATH "${CMAKE_CURRENT_SOURCE_DIR}/precompiled_bitcode.cc.in") +# add_arrow_lib will look for this not yet existing file, so write it +if(NOT EXISTS ${GANDIVA_PRECOMPILED_CC_PATH}) + configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_CC_PATH}) +endif() set(SRC_FILES annotator.cc @@ -73,7 +70,8 @@ set(SRC_FILES selection_vector.cc tree_expr_builder.cc to_date_holder.cc - ${BC_FILE_PATH_CC}) + ${GANDIVA_PRECOMPILED_CC_PATH} +) set(GANDIVA_SHARED_PRIVATE_LINK_LIBS arrow_shared LLVM::LLVM_INTERFACE ${RE2_LIBRARY}) diff --git a/cpp/src/gandiva/configuration.cc b/cpp/src/gandiva/configuration.cc index df1a13d570c..f54f7a91804 100644 --- a/cpp/src/gandiva/configuration.cc +++ b/cpp/src/gandiva/configuration.cc @@ -24,13 +24,13 @@ namespace gandiva { const std::shared_ptr ConfigurationBuilder::default_configuration_ = InitDefaultConfig(); -std::size_t Configuration::Hash() const { - boost::hash string_hash; - return string_hash(byte_code_file_path_); -} +// std::size_t Configuration::Hash() const { +// boost::hash string_hash; +// return string_hash(byte_code_file_path_); +// } bool Configuration::operator==(const Configuration& other) const { - return other.byte_code_file_path() == byte_code_file_path(); + return other.precompiled_bitcode() == precompiled_bitcode(); } bool Configuration::operator!=(const Configuration& other) const { diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h index 480a95e9274..11697f6996a 100644 --- a/cpp/src/gandiva/configuration.h +++ b/cpp/src/gandiva/configuration.h @@ -27,7 +27,8 @@ namespace gandiva { GANDIVA_EXPORT -extern const char kByteCodeFilePath[]; +extern const char kPrecompiledBitcode[]; +extern const unsigned kPrecompiledBitcodeSize; class ConfigurationBuilder; /// \brief runtime config for gandiva @@ -38,17 +39,17 @@ class GANDIVA_EXPORT Configuration { public: friend class ConfigurationBuilder; - const std::string& byte_code_file_path() const { return byte_code_file_path_; } + const std::string& precompiled_bitcode() const { return precompiled_bitcode_; } - std::size_t Hash() const; + // std::size_t Hash() const; bool operator==(const Configuration& other) const; bool operator!=(const Configuration& other) const; private: - explicit Configuration(const std::string& byte_code_file_path) - : byte_code_file_path_(byte_code_file_path) {} + explicit Configuration(const std::string& precompiled_bitcode) + : precompiled_bitcode_(precompiled_bitcode) {} - const std::string byte_code_file_path_; + const std::string precompiled_bitcode_; }; /// \brief configuration builder for gandiva @@ -57,15 +58,13 @@ class GANDIVA_EXPORT Configuration { /// to override specific values and build a custom instance class GANDIVA_EXPORT ConfigurationBuilder { public: - ConfigurationBuilder() : byte_code_file_path_(kByteCodeFilePath) {} - - ConfigurationBuilder& set_byte_code_file_path(const std::string& byte_code_file_path) { - byte_code_file_path_ = byte_code_file_path; + ConfigurationBuilder& set_precompiled_bitcode(const std::string& precompiled_bitcode) { + precompiled_bitcode_ = precompiled_bitcode; return *this; } std::shared_ptr build() { - std::shared_ptr configuration(new Configuration(byte_code_file_path_)); + std::shared_ptr configuration(new Configuration(precompiled_bitcode_)); return configuration; } @@ -74,10 +73,11 @@ class GANDIVA_EXPORT ConfigurationBuilder { } private: - std::string byte_code_file_path_; + std::string precompiled_bitcode_; static std::shared_ptr InitDefaultConfig() { - std::shared_ptr configuration(new Configuration(kByteCodeFilePath)); + const std::string bitcode(kPrecompiledBitcode, kPrecompiledBitcodeSize); + std::shared_ptr configuration(new Configuration(bitcode)); return configuration; } diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index c816ec76ff4..50b9220e96b 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -114,7 +114,7 @@ Status Engine::Make(std::shared_ptr config, // Add mappings for functions that can be accessed from LLVM/IR module. engine_obj->AddGlobalMappings(); - auto status = engine_obj->LoadPreCompiledIRFiles(config->byte_code_file_path()); + auto status = engine_obj->LoadPreCompiledIR(config->precompiled_bitcode()); ARROW_RETURN_NOT_OK(status); // Add decimal functions @@ -126,14 +126,13 @@ Status Engine::Make(std::shared_ptr config, } // Handling for pre-compiled IR libraries. -Status Engine::LoadPreCompiledIRFiles(const std::string& byte_code_file_path) { +Status Engine::LoadPreCompiledIR(const std::string& precompiled_bitcode) { /// Read from file into memory buffer. llvm::ErrorOr> buffer_or_error = - llvm::MemoryBuffer::getFile(byte_code_file_path); - ARROW_RETURN_IF( - !buffer_or_error, - Status::CodeGenError("Could not load module from IR ", byte_code_file_path, ": ", - buffer_or_error.getError().message())); + llvm::MemoryBuffer::getMemBuffer(precompiled_bitcode); + ARROW_RETURN_IF(!buffer_or_error, + Status::CodeGenError("Could not load module from IR: ", + buffer_or_error.getError().message())); std::unique_ptr buffer = move(buffer_or_error.get()); diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 7a976d5f926..16e546d04c5 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -79,8 +79,8 @@ class GANDIVA_EXPORT Engine { llvm::ExecutionEngine& execution_engine() { return *execution_engine_.get(); } - /// load pre-compiled IR modules and merge them into the main module. - Status LoadPreCompiledIRFiles(const std::string& byte_code_file_path); + /// load pre-compiled IR modules from string and merge them into the main module. + Status LoadPreCompiledIR(const std::string& precompiled_bitcode); // Create and add mappings for cpp functions that can be accessed from LLVM. void AddGlobalMappings(); diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 8e2f4989676..68d1dd0e6c9 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -67,12 +67,29 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS}) endforeach() # link all of the bitcode files into a single bitcode file. -add_custom_command(OUTPUT ${GANDIVA_BC_OUTPUT_PATH} - COMMAND ${LLVM_LINK_EXECUTABLE} -o ${GANDIVA_BC_OUTPUT_PATH} - ${BC_FILES} +add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH} + COMMAND ${LLVM_LINK_EXECUTABLE} -o ${GANDIVA_PRECOMPILED_BC_PATH} ${BC_FILES} DEPENDS ${BC_FILES}) -add_custom_target(precompiled ALL DEPENDS ${GANDIVA_BC_OUTPUT_PATH}) +# write a cmake script to replace precompiled bitcode file's content into a .cc file + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" "\ + file(READ \"${GANDIVA_PRECOMPILED_BC_PATH}\" HEXCONTENT HEX) + string(REGEX REPLACE + \"(..)(..)(..)(..)(..)\" + \"'\\\\\\\\x\\\\1','\\\\\\\\x\\\\2','\\\\\\\\x\\\\3','\\\\\\\\x\\\\4','\\\\\\\\x\\\\5',\" + GANDIVA_PRECOMPILED_BITCODE_CHARS + \"\${HEXCONTENT}\") + configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH} + ${GANDIVA_PRECOMPILED_CC_PATH}) +") + +# add the previous command to the execution chain +add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH} + COMMAND ${CMAKE_COMMAND} -P "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" + DEPENDS ${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_BC_PATH}) + +add_custom_target(precompiled ALL DEPENDS ${GANDIVA_PRECOMPILED_CC_PATH}) function(add_precompiled_unit_test REL_TEST_NAME) get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) diff --git a/cpp/src/gandiva/bc_file_path.cc.in b/cpp/src/gandiva/precompiled_bitcode.cc.in similarity index 80% rename from cpp/src/gandiva/bc_file_path.cc.in rename to cpp/src/gandiva/precompiled_bitcode.cc.in index 54e81ca2bfa..a3ca927db57 100644 --- a/cpp/src/gandiva/bc_file_path.cc.in +++ b/cpp/src/gandiva/precompiled_bitcode.cc.in @@ -17,7 +17,8 @@ namespace gandiva { -// Path to the byte-code file. -extern const char kByteCodeFilePath[] = "${GANDIVA_BC_INSTALL_PATH}"; +// Content of precompiled bitcode file. +extern const char kPrecompiledBitcode[] = { ${GANDIVA_PRECOMPILED_BITCODE_CHARS} }; +extern const unsigned kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode); } // namespace gandiva diff --git a/cpp/src/gandiva/projector_cache_key.h b/cpp/src/gandiva/projector_cache_key.h index 26da5288e5d..d4ae5274af3 100644 --- a/cpp/src/gandiva/projector_cache_key.h +++ b/cpp/src/gandiva/projector_cache_key.h @@ -41,7 +41,7 @@ class ProjectorCacheKey { boost::hash_combine(result, expr_as_string); UpdateUniqifier(expr_as_string); } - boost::hash_combine(result, configuration->Hash()); + // boost::hash_combine(result, configuration->Hash()); boost::hash_combine(result, schema_->ToString()); boost::hash_combine(result, uniqifier_); hash_code_ = result; diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index ba0e63292f4..f9b803eea85 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -88,17 +88,17 @@ TEST_F(TestProjector, TestProjectCache) { // if configuration is different, should return a new projector. // build a new path by replacing the first '/' with '//' - std::string alt_path(GANDIVA_BYTE_COMPILE_FILE_PATH); - auto pos = alt_path.find('/', 0); - EXPECT_NE(pos, std::string::npos); - alt_path.replace(pos, 1, "//"); - auto other_configuration = - ConfigurationBuilder().set_byte_code_file_path(alt_path).build(); - std::shared_ptr should_be_new_projector2; - status = Projector::Make(schema, {sum_expr, sub_expr}, other_configuration, - &should_be_new_projector2); - ASSERT_OK(status); - EXPECT_NE(projector, should_be_new_projector2); + // std::string alt_path(GANDIVA_BYTE_COMPILE_FILE_PATH); + // auto pos = alt_path.find('/', 0); + // EXPECT_NE(pos, std::string::npos); + // alt_path.replace(pos, 1, "//"); + // auto other_configuration = + // ConfigurationBuilder().set_byte_code_file_path(alt_path).build(); + // std::shared_ptr should_be_new_projector2; + // status = Projector::Make(schema, {sum_expr, sub_expr}, other_configuration, + // &should_be_new_projector2); + // ASSERT_OK(status); + // EXPECT_NE(projector, should_be_new_projector2); } TEST_F(TestProjector, TestProjectCacheFieldNames) { diff --git a/cpp/src/gandiva/tests/test_util.h b/cpp/src/gandiva/tests/test_util.h index af164125886..e841e6a0ebf 100644 --- a/cpp/src/gandiva/tests/test_util.h +++ b/cpp/src/gandiva/tests/test_util.h @@ -89,8 +89,7 @@ static ArrayPtr MakeArrowTypeArray(const std::shared_ptr& type, std::shared_ptr TestConfiguration() { auto builder = ConfigurationBuilder(); - builder.set_byte_code_file_path(GANDIVA_BYTE_COMPILE_FILE_PATH); - return builder.build(); + return builder.DefaultConfiguration(); } } // namespace gandiva From f2205d01833f1c7ef12e6c6643f4c3821b1ca053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 13:47:54 +0100 Subject: [PATCH 05/34] gandiva jni --- cpp/src/gandiva/jni/config_builder.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/gandiva/jni/config_builder.cc b/cpp/src/gandiva/jni/config_builder.cc index b224279c1a6..18f24fbe17b 100644 --- a/cpp/src/gandiva/jni/config_builder.cc +++ b/cpp/src/gandiva/jni/config_builder.cc @@ -34,16 +34,16 @@ using gandiva::ConfigurationBuilder; JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_ConfigurationBuilder_buildConfigInstance( JNIEnv* env, jobject configuration) { - jstring byte_code_file_path = + jstring precompiled_bitcode = (jstring)env->CallObjectMethod(configuration, byte_code_accessor_method_id_, 0); ConfigurationBuilder configuration_builder; - if (byte_code_file_path != nullptr) { - const char* byte_code_file_path_cpp = env->GetStringUTFChars(byte_code_file_path, 0); - configuration_builder.set_byte_code_file_path(byte_code_file_path_cpp); - env->ReleaseStringUTFChars(byte_code_file_path, byte_code_file_path_cpp); + if (precompiled_bitcode != nullptr) { + const char* precompiled_bitcode_cpp = env->GetStringUTFChars(precompiled_bitcode, 0); + configuration_builder.set_precompiled_bitcode(precompiled_bitcode_cpp); + env->ReleaseStringUTFChars(precompiled_bitcode, precompiled_bitcode_cpp); } std::shared_ptr config = configuration_builder.build(); - env->DeleteLocalRef(byte_code_file_path); + env->DeleteLocalRef(precompiled_bitcode); return ConfigHolder::MapInsert(config); } From 19200c3074deb95335790b387b6c37f698e3b908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 14:45:46 +0100 Subject: [PATCH 06/34] don't run wheel tests twice; cmake format --- ci/travis_script_manylinux.sh | 1 + cpp/src/gandiva/CMakeLists.txt | 6 ++-- cpp/src/gandiva/precompiled/CMakeLists.txt | 14 +++++--- python/manylinux1/build_arrow.sh | 38 +++++++++++----------- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 239f0a65c97..a8afc86ab6d 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -23,6 +23,7 @@ set -ex pushd python/manylinux1 docker run --shm-size=2g --rm \ -e PYARROW_PARALLEL=3 \ + -e PYTHON_VERSIONS=$PYTHON_VERSIONS \ -v $PWD:/io \ -v $PWD/../../:/arrow \ quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 \ diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index b0dc3bf1a83..629fbcd2732 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -30,7 +30,8 @@ find_package(LLVM) # Set the path where the bitcode file generated, see precompiled/CMakeLists.txt set(GANDIVA_PRECOMPILED_BC_PATH "${CMAKE_CURRENT_BINARY_DIR}/irhelpers.bc") set(GANDIVA_PRECOMPILED_CC_PATH "${CMAKE_CURRENT_BINARY_DIR}/precompiled_bitcode.cc") -set(GANDIVA_PRECOMPILED_CC_IN_PATH "${CMAKE_CURRENT_SOURCE_DIR}/precompiled_bitcode.cc.in") +set(GANDIVA_PRECOMPILED_CC_IN_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/precompiled_bitcode.cc.in") # add_arrow_lib will look for this not yet existing file, so write it if(NOT EXISTS ${GANDIVA_PRECOMPILED_CC_PATH}) configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_CC_PATH}) @@ -70,8 +71,7 @@ set(SRC_FILES selection_vector.cc tree_expr_builder.cc to_date_holder.cc - ${GANDIVA_PRECOMPILED_CC_PATH} -) + ${GANDIVA_PRECOMPILED_CC_PATH}) set(GANDIVA_SHARED_PRIVATE_LINK_LIBS arrow_shared LLVM::LLVM_INTERFACE ${RE2_LIBRARY}) diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 68d1dd0e6c9..b9c5db4384a 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -68,12 +68,16 @@ endforeach() # link all of the bitcode files into a single bitcode file. add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH} - COMMAND ${LLVM_LINK_EXECUTABLE} -o ${GANDIVA_PRECOMPILED_BC_PATH} ${BC_FILES} + COMMAND ${LLVM_LINK_EXECUTABLE} -o ${GANDIVA_PRECOMPILED_BC_PATH} + ${BC_FILES} DEPENDS ${BC_FILES}) # write a cmake script to replace precompiled bitcode file's content into a .cc file -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" "\ +file( + WRITE + "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" + "\ file(READ \"${GANDIVA_PRECOMPILED_BC_PATH}\" HEXCONTENT HEX) string(REGEX REPLACE \"(..)(..)(..)(..)(..)\" @@ -86,8 +90,10 @@ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" "\ # add the previous command to the execution chain add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH} - COMMAND ${CMAKE_COMMAND} -P "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" - DEPENDS ${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_BC_PATH}) + COMMAND ${CMAKE_COMMAND} -P + "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" + DEPENDS ${GANDIVA_PRECOMPILED_CC_IN_PATH} + ${GANDIVA_PRECOMPILED_BC_PATH}) add_custom_target(precompiled ALL DEPENDS ${GANDIVA_PRECOMPILED_CC_PATH}) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 75537a64e73..884ca365e13 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -117,25 +117,25 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do mkdir -p repaired_wheels/ auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ - echo "=== (${PYTHON}) Testing manylinux1 wheel ===" - source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate - pip install repaired_wheels/*.whl - - if [ $PYTHON != "2.7" ]; then - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.gandiva" - fi - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.orc" - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet" - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma" - - echo "=== (${PYTHON}) Install modules required for testing ===" - pip install -r requirements-test.txt - - # The TensorFlow test will be skipped here, since TensorFlow is not - # manylinux1 compatible; however, the wheels will support TensorFlow on - # a TensorFlow compatible system - py.test -v -r sxX --durations=15 --parquet ${VIRTUAL_ENV}/lib/*/site-packages/pyarrow - deactivate + # echo "=== (${PYTHON}) Testing manylinux1 wheel ===" + # source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate + # pip install repaired_wheels/*.whl + # + # if [ $PYTHON != "2.7" ]; then + # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.gandiva" + # fi + # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.orc" + # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet" + # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma" + # + # echo "=== (${PYTHON}) Install modules required for testing ===" + # pip install -r requirements-test.txt + # + # # The TensorFlow test will be skipped here, since TensorFlow is not + # # manylinux1 compatible; however, the wheels will support TensorFlow on + # # a TensorFlow compatible system + # py.test -v -r sxX --durations=15 --parquet ${VIRTUAL_ENV}/lib/*/site-packages/pyarrow + # deactivate mv repaired_wheels/*.whl /io/dist mv dist/*.tar.gz /io/dist From cb6962519c82e5b5312abfba8a11f8caa8ef53c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 15:37:49 +0100 Subject: [PATCH 07/34] silent maven download msgs --- ci/travis_script_gandiva_java.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/travis_script_gandiva_java.sh b/ci/travis_script_gandiva_java.sh index 387be9a092b..9d5871f69fc 100755 --- a/ci/travis_script_gandiva_java.sh +++ b/ci/travis_script_gandiva_java.sh @@ -22,6 +22,8 @@ set -e source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh JAVA_DIR=${TRAVIS_BUILD_DIR}/java +export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + pushd $JAVA_DIR # build with gandiva profile From 169f43ace310b3cc11a5be901f8a4224642e21fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 15:53:06 +0100 Subject: [PATCH 08/34] build gandiva in cpp docker image --- ci/docker_build_cpp.sh | 2 ++ cpp/Dockerfile | 1 + 2 files changed, 3 insertions(+) diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh index 450dc870249..57da66534d4 100755 --- a/ci/docker_build_cpp.sh +++ b/ci/docker_build_cpp.sh @@ -34,6 +34,8 @@ cmake -GNinja \ -DARROW_PARQUET=${ARROW_PARQUET:-ON} \ -DARROW_HDFS=${ARROW_HDFS:-OFF} \ -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-ON} \ + -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ diff --git a/cpp/Dockerfile b/cpp/Dockerfile index 17d332d22be..a8e2f582a48 100644 --- a/cpp/Dockerfile +++ b/cpp/Dockerfile @@ -23,6 +23,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y -q --no-install-recommends \ ca-certificates \ ccache \ + clang-7 \ g++ \ gcc \ git \ From b0b111754907ad4ffed369f1231625a13934233c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 16:36:43 +0100 Subject: [PATCH 09/34] conda llvmdev --- ci/docker_build_cpp.sh | 2 +- cpp/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh index 57da66534d4..5bf6094ae2a 100755 --- a/ci/docker_build_cpp.sh +++ b/ci/docker_build_cpp.sh @@ -34,7 +34,7 @@ cmake -GNinja \ -DARROW_PARQUET=${ARROW_PARQUET:-ON} \ -DARROW_HDFS=${ARROW_HDFS:-OFF} \ -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-ON} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ diff --git a/cpp/Dockerfile b/cpp/Dockerfile index a8e2f582a48..4d177524e7d 100644 --- a/cpp/Dockerfile +++ b/cpp/Dockerfile @@ -23,7 +23,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y -q --no-install-recommends \ ca-certificates \ ccache \ - clang-7 \ g++ \ gcc \ git \ @@ -44,11 +43,12 @@ RUN arrow/ci/docker_install_conda.sh && \ conda install -q -c conda-forge \ --file arrow/ci/conda_env_cpp.yml \ --file arrow/ci/conda_env_unix.yml \ - $EXTRA_CONDA_PKGS && \ + llvmdev $EXTRA_CONDA_PKGS && \ conda clean --all ENV CC=gcc \ CXX=g++ \ + ARROW_GANDIVA=ON \ ARROW_BUILD_TESTS=ON \ ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX \ ARROW_HOME=$CONDA_PREFIX \ From d480c83c69e34c1dbc7cdc7ea51348a87271052c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 15 Feb 2019 17:00:00 +0100 Subject: [PATCH 10/34] use const string ref for now --- cpp/src/gandiva/configuration.h | 10 +++++----- cpp/src/gandiva/precompiled_bitcode.cc.in | 7 +++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h index 11697f6996a..0515104b1da 100644 --- a/cpp/src/gandiva/configuration.h +++ b/cpp/src/gandiva/configuration.h @@ -27,8 +27,7 @@ namespace gandiva { GANDIVA_EXPORT -extern const char kPrecompiledBitcode[]; -extern const unsigned kPrecompiledBitcodeSize; +extern const std::string kPrecompiledBitcode; class ConfigurationBuilder; /// \brief runtime config for gandiva @@ -49,7 +48,7 @@ class GANDIVA_EXPORT Configuration { explicit Configuration(const std::string& precompiled_bitcode) : precompiled_bitcode_(precompiled_bitcode) {} - const std::string precompiled_bitcode_; + const std::string& precompiled_bitcode_; }; /// \brief configuration builder for gandiva @@ -58,6 +57,8 @@ class GANDIVA_EXPORT Configuration { /// to override specific values and build a custom instance class GANDIVA_EXPORT ConfigurationBuilder { public: + ConfigurationBuilder() : precompiled_bitcode_(kPrecompiledBitcode) {} + ConfigurationBuilder& set_precompiled_bitcode(const std::string& precompiled_bitcode) { precompiled_bitcode_ = precompiled_bitcode; return *this; @@ -76,8 +77,7 @@ class GANDIVA_EXPORT ConfigurationBuilder { std::string precompiled_bitcode_; static std::shared_ptr InitDefaultConfig() { - const std::string bitcode(kPrecompiledBitcode, kPrecompiledBitcodeSize); - std::shared_ptr configuration(new Configuration(bitcode)); + std::shared_ptr configuration(new Configuration(kPrecompiledBitcode)); return configuration; } diff --git a/cpp/src/gandiva/precompiled_bitcode.cc.in b/cpp/src/gandiva/precompiled_bitcode.cc.in index a3ca927db57..ee945e88935 100644 --- a/cpp/src/gandiva/precompiled_bitcode.cc.in +++ b/cpp/src/gandiva/precompiled_bitcode.cc.in @@ -15,10 +15,13 @@ // specific language governing permissions and limitations // under the License. +#include + namespace gandiva { // Content of precompiled bitcode file. -extern const char kPrecompiledBitcode[] = { ${GANDIVA_PRECOMPILED_BITCODE_CHARS} }; -extern const unsigned kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode); +// extern const char kPrecompiledBitcode[] = ; +// extern const unsigned kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode); +extern const std::string kPrecompiledBitcode({ ${GANDIVA_PRECOMPILED_BITCODE_CHARS} }); } // namespace gandiva From 42391b13c70e723314b4debfe7c2051b717e2a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Feb 2019 20:47:52 +0100 Subject: [PATCH 11/34] don't pass precompiled bitcode all around the constructors --- cpp/src/gandiva/configuration.cc | 9 +++---- cpp/src/gandiva/configuration.h | 26 +++---------------- cpp/src/gandiva/engine.cc | 5 +++- cpp/src/gandiva/jni/config_builder.cc | 8 ------ cpp/src/gandiva/projector_cache_key.h | 2 +- .../arrow/gandiva/evaluator/JniLoader.java | 20 +++----------- 6 files changed, 15 insertions(+), 55 deletions(-) diff --git a/cpp/src/gandiva/configuration.cc b/cpp/src/gandiva/configuration.cc index f54f7a91804..0c6a33fc00e 100644 --- a/cpp/src/gandiva/configuration.cc +++ b/cpp/src/gandiva/configuration.cc @@ -24,13 +24,12 @@ namespace gandiva { const std::shared_ptr ConfigurationBuilder::default_configuration_ = InitDefaultConfig(); -// std::size_t Configuration::Hash() const { -// boost::hash string_hash; -// return string_hash(byte_code_file_path_); -// } +std::size_t Configuration::Hash() const { + return 0; // dummy for now, no configuration properties yet +} bool Configuration::operator==(const Configuration& other) const { - return other.precompiled_bitcode() == precompiled_bitcode(); + return true; // always true, no configuration properties yet } bool Configuration::operator!=(const Configuration& other) const { diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h index 0515104b1da..7fb0d63cf57 100644 --- a/cpp/src/gandiva/configuration.h +++ b/cpp/src/gandiva/configuration.h @@ -26,9 +26,6 @@ namespace gandiva { -GANDIVA_EXPORT -extern const std::string kPrecompiledBitcode; - class ConfigurationBuilder; /// \brief runtime config for gandiva /// @@ -38,17 +35,9 @@ class GANDIVA_EXPORT Configuration { public: friend class ConfigurationBuilder; - const std::string& precompiled_bitcode() const { return precompiled_bitcode_; } - - // std::size_t Hash() const; + std::size_t Hash() const; bool operator==(const Configuration& other) const; bool operator!=(const Configuration& other) const; - - private: - explicit Configuration(const std::string& precompiled_bitcode) - : precompiled_bitcode_(precompiled_bitcode) {} - - const std::string& precompiled_bitcode_; }; /// \brief configuration builder for gandiva @@ -57,15 +46,8 @@ class GANDIVA_EXPORT Configuration { /// to override specific values and build a custom instance class GANDIVA_EXPORT ConfigurationBuilder { public: - ConfigurationBuilder() : precompiled_bitcode_(kPrecompiledBitcode) {} - - ConfigurationBuilder& set_precompiled_bitcode(const std::string& precompiled_bitcode) { - precompiled_bitcode_ = precompiled_bitcode; - return *this; - } - std::shared_ptr build() { - std::shared_ptr configuration(new Configuration(precompiled_bitcode_)); + std::shared_ptr configuration(new Configuration()); return configuration; } @@ -74,10 +56,8 @@ class GANDIVA_EXPORT ConfigurationBuilder { } private: - std::string precompiled_bitcode_; - static std::shared_ptr InitDefaultConfig() { - std::shared_ptr configuration(new Configuration(kPrecompiledBitcode)); + std::shared_ptr configuration(new Configuration()); return configuration; } diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 50b9220e96b..81e651683dc 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -66,6 +66,9 @@ namespace gandiva { +GANDIVA_EXPORT +extern const std::string kPrecompiledBitcode; + std::once_flag init_once_flag; bool Engine::init_once_done_ = false; @@ -114,7 +117,7 @@ Status Engine::Make(std::shared_ptr config, // Add mappings for functions that can be accessed from LLVM/IR module. engine_obj->AddGlobalMappings(); - auto status = engine_obj->LoadPreCompiledIR(config->precompiled_bitcode()); + auto status = engine_obj->LoadPreCompiledIR(kPrecompiledBitcode); ARROW_RETURN_NOT_OK(status); // Add decimal functions diff --git a/cpp/src/gandiva/jni/config_builder.cc b/cpp/src/gandiva/jni/config_builder.cc index 18f24fbe17b..f2694bbee6d 100644 --- a/cpp/src/gandiva/jni/config_builder.cc +++ b/cpp/src/gandiva/jni/config_builder.cc @@ -34,16 +34,8 @@ using gandiva::ConfigurationBuilder; JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_ConfigurationBuilder_buildConfigInstance( JNIEnv* env, jobject configuration) { - jstring precompiled_bitcode = - (jstring)env->CallObjectMethod(configuration, byte_code_accessor_method_id_, 0); ConfigurationBuilder configuration_builder; - if (precompiled_bitcode != nullptr) { - const char* precompiled_bitcode_cpp = env->GetStringUTFChars(precompiled_bitcode, 0); - configuration_builder.set_precompiled_bitcode(precompiled_bitcode_cpp); - env->ReleaseStringUTFChars(precompiled_bitcode, precompiled_bitcode_cpp); - } std::shared_ptr config = configuration_builder.build(); - env->DeleteLocalRef(precompiled_bitcode); return ConfigHolder::MapInsert(config); } diff --git a/cpp/src/gandiva/projector_cache_key.h b/cpp/src/gandiva/projector_cache_key.h index d4ae5274af3..26da5288e5d 100644 --- a/cpp/src/gandiva/projector_cache_key.h +++ b/cpp/src/gandiva/projector_cache_key.h @@ -41,7 +41,7 @@ class ProjectorCacheKey { boost::hash_combine(result, expr_as_string); UpdateUniqifier(expr_as_string); } - // boost::hash_combine(result, configuration->Hash()); + boost::hash_combine(result, configuration->Hash()); boost::hash_combine(result, schema_->ToString()); boost::hash_combine(result, uniqifier_); hash_code_ = result; diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java index ccb53070494..95e3cadbd99 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java @@ -32,16 +32,13 @@ */ class JniLoader { private static final String LIBRARY_NAME = "gandiva_jni"; - private static final String IRHELPERS_BC = "irhelpers.bc"; private static volatile JniLoader INSTANCE; private static volatile long defaultConfiguration = 0L; - private final String byteCodeFilePath; private final JniWrapper wrapper; - private JniLoader(String byteCodeFilePath) { - this.byteCodeFilePath = byteCodeFilePath; + private JniLoader() { this.wrapper = new JniWrapper(); } @@ -60,8 +57,7 @@ private static JniLoader setupInstance() throws GandivaException { try { String tempDir = System.getProperty("java.io.tmpdir"); loadGandivaLibraryFromJar(tempDir); - File byteCodeFile = moveFileFromJarToTemp(tempDir, IRHELPERS_BC); - return new JniLoader(byteCodeFile.getAbsolutePath()); + return new JniLoader(); } catch (IOException ioException) { throw new GandivaException("unable to create native instance", ioException); } @@ -107,13 +103,6 @@ private static File setupFile(String tmpDir, String libraryToLoad) return temp; } - /** - * Returns the byte code file path extracted from jar. - */ - public String getByteCodeFilePath() { - return byteCodeFilePath; - } - /** * Returns the jni wrapper. */ @@ -130,11 +119,8 @@ static long getDefaultConfiguration() throws GandivaException { if (defaultConfiguration == 0L) { synchronized (ConfigurationBuilder.class) { if (defaultConfiguration == 0L) { - String defaultByteCodeFilePath = JniLoader.getInstance().getByteCodeFilePath(); - defaultConfiguration = new ConfigurationBuilder() - .withByteCodeFilePath(defaultByteCodeFilePath) - .buildConfigInstance(); + .buildConfigInstance(); } } } From 5841dcdde8a8780ffa0b3de24462769ea0a02fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Feb 2019 20:57:33 +0100 Subject: [PATCH 12/34] remove commented code --- cpp/src/gandiva/tests/projector_test.cc | 15 --------------- python/manylinux1/build_arrow.sh | 21 +-------------------- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index f9b803eea85..8c0b9ddccb0 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -84,21 +84,6 @@ TEST_F(TestProjector, TestProjectCache) { &cached_projector); ASSERT_OK(status); EXPECT_EQ(cached_projector, projector); - - // if configuration is different, should return a new projector. - - // build a new path by replacing the first '/' with '//' - // std::string alt_path(GANDIVA_BYTE_COMPILE_FILE_PATH); - // auto pos = alt_path.find('/', 0); - // EXPECT_NE(pos, std::string::npos); - // alt_path.replace(pos, 1, "//"); - // auto other_configuration = - // ConfigurationBuilder().set_byte_code_file_path(alt_path).build(); - // std::shared_ptr should_be_new_projector2; - // status = Projector::Make(schema, {sum_expr, sub_expr}, other_configuration, - // &should_be_new_projector2); - // ASSERT_OK(status); - // EXPECT_NE(projector, should_be_new_projector2); } TEST_F(TestProjector, TestProjectCacheFieldNames) { diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 884ca365e13..ee4570e90a5 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -117,26 +117,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do mkdir -p repaired_wheels/ auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ - # echo "=== (${PYTHON}) Testing manylinux1 wheel ===" - # source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate - # pip install repaired_wheels/*.whl - # - # if [ $PYTHON != "2.7" ]; then - # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.gandiva" - # fi - # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.orc" - # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet" - # PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma" - # - # echo "=== (${PYTHON}) Install modules required for testing ===" - # pip install -r requirements-test.txt - # - # # The TensorFlow test will be skipped here, since TensorFlow is not - # # manylinux1 compatible; however, the wheels will support TensorFlow on - # # a TensorFlow compatible system - # py.test -v -r sxX --durations=15 --parquet ${VIRTUAL_ENV}/lib/*/site-packages/pyarrow - # deactivate - + # Testing happens outsite of the build to prevent issues like ARROW-4372 mv repaired_wheels/*.whl /io/dist mv dist/*.tar.gz /io/dist done From fa195294a29b48b53a4bf43c80bb2974da3fc9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 02:05:31 +0100 Subject: [PATCH 13/34] properly construct llvm:StringRef --- cpp/src/gandiva/engine.cc | 12 ++++++++---- cpp/src/gandiva/engine.h | 5 +++-- cpp/src/gandiva/precompiled/CMakeLists.txt | 4 ++-- cpp/src/gandiva/precompiled_bitcode.cc.in | 5 ++--- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 81e651683dc..c4c19e74f72 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -67,7 +67,8 @@ namespace gandiva { GANDIVA_EXPORT -extern const std::string kPrecompiledBitcode; +extern const char kPrecompiledBitcode[]; +extern const size_t kPrecompiledBitcodeSize; std::once_flag init_once_flag; @@ -117,7 +118,7 @@ Status Engine::Make(std::shared_ptr config, // Add mappings for functions that can be accessed from LLVM/IR module. engine_obj->AddGlobalMappings(); - auto status = engine_obj->LoadPreCompiledIR(kPrecompiledBitcode); + auto status = engine_obj->LoadPreCompiledIR(); ARROW_RETURN_NOT_OK(status); // Add decimal functions @@ -129,10 +130,13 @@ Status Engine::Make(std::shared_ptr config, } // Handling for pre-compiled IR libraries. -Status Engine::LoadPreCompiledIR(const std::string& precompiled_bitcode) { +Status Engine::LoadPreCompiledIR() { + auto bitcode = llvm::StringRef(kPrecompiledBitcode, kPrecompiledBitcodeSize); + /// Read from file into memory buffer. llvm::ErrorOr> buffer_or_error = - llvm::MemoryBuffer::getMemBuffer(precompiled_bitcode); + llvm::MemoryBuffer::getMemBuffer(bitcode); + ARROW_RETURN_IF(!buffer_or_error, Status::CodeGenError("Could not load module from IR: ", buffer_or_error.getError().message())); diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 16e546d04c5..3b46f62f5ef 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -79,8 +79,9 @@ class GANDIVA_EXPORT Engine { llvm::ExecutionEngine& execution_engine() { return *execution_engine_.get(); } - /// load pre-compiled IR modules from string and merge them into the main module. - Status LoadPreCompiledIR(const std::string& precompiled_bitcode); + /// load pre-compiled IR modules from precompiled_bitcode.cc and merge them into + /// the main module. + Status LoadPreCompiledIR(); // Create and add mappings for cpp functions that can be accessed from LLVM. void AddGlobalMappings(); diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index b9c5db4384a..0fac4eb8620 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -73,7 +73,6 @@ add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH} DEPENDS ${BC_FILES}) # write a cmake script to replace precompiled bitcode file's content into a .cc file - file( WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" @@ -95,7 +94,8 @@ add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH} DEPENDS ${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_BC_PATH}) -add_custom_target(precompiled ALL DEPENDS ${GANDIVA_PRECOMPILED_CC_PATH}) +add_custom_target(precompiled ALL + DEPENDS ${GANDIVA_PRECOMPILED_BC_PATH} ${GANDIVA_PRECOMPILED_CC_PATH}) function(add_precompiled_unit_test REL_TEST_NAME) get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) diff --git a/cpp/src/gandiva/precompiled_bitcode.cc.in b/cpp/src/gandiva/precompiled_bitcode.cc.in index ee945e88935..1d5985aa48d 100644 --- a/cpp/src/gandiva/precompiled_bitcode.cc.in +++ b/cpp/src/gandiva/precompiled_bitcode.cc.in @@ -20,8 +20,7 @@ namespace gandiva { // Content of precompiled bitcode file. -// extern const char kPrecompiledBitcode[] = ; -// extern const unsigned kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode); -extern const std::string kPrecompiledBitcode({ ${GANDIVA_PRECOMPILED_BITCODE_CHARS} }); +extern const char kPrecompiledBitcode[] = { ${GANDIVA_PRECOMPILED_BITCODE_CHARS} }; +extern const size_t kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode); } // namespace gandiva From 7deb3593e0b519ebfcce24e296e1c20a904daf7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 03:23:06 +0100 Subject: [PATCH 14/34] fix REGEX; remove byteCodeFilePath from java configuration object --- cpp/src/gandiva/jni/jni_common.cc | 6 ------ cpp/src/gandiva/precompiled/CMakeLists.txt | 4 ++-- .../gandiva/evaluator/ConfigurationBuilder.java | 12 +----------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/cpp/src/gandiva/jni/jni_common.cc b/cpp/src/gandiva/jni/jni_common.cc index 339b0cd4e5a..1ba3aad16cd 100644 --- a/cpp/src/gandiva/jni/jni_common.cc +++ b/cpp/src/gandiva/jni/jni_common.cc @@ -68,7 +68,6 @@ static jint JNI_VERSION = JNI_VERSION_1_6; // extern refs - initialized for other modules. jclass configuration_builder_class_; -jmethodID byte_code_accessor_method_id_; // refs for self. static jclass gandiva_exception_; @@ -93,11 +92,6 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { gandiva_exception_ = (jclass)env->NewGlobalRef(localExceptionClass); env->DeleteLocalRef(localExceptionClass); - const char method_name[] = "getByteCodeFilePath"; - const char return_type[] = "()Ljava/lang/String;"; - byte_code_accessor_method_id_ = - env->GetMethodID(configuration_builder_class_, method_name, return_type); - env->ExceptionDescribe(); return JNI_VERSION; diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 0fac4eb8620..cbb38842718 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -79,8 +79,8 @@ file( "\ file(READ \"${GANDIVA_PRECOMPILED_BC_PATH}\" HEXCONTENT HEX) string(REGEX REPLACE - \"(..)(..)(..)(..)(..)\" - \"'\\\\\\\\x\\\\1','\\\\\\\\x\\\\2','\\\\\\\\x\\\\3','\\\\\\\\x\\\\4','\\\\\\\\x\\\\5',\" + \"([0-9a-f][0-9a-f])\" + \"'\\\\\\\\x\\\\1',\" GANDIVA_PRECOMPILED_BITCODE_CHARS \"\${HEXCONTENT}\") configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH} diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java index 46deee95fa7..f0e8c0d826a 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java @@ -22,18 +22,8 @@ */ public class ConfigurationBuilder { - private String byteCodeFilePath = ""; - - public ConfigurationBuilder withByteCodeFilePath(final String byteCodeFilePath) { - this.byteCodeFilePath = byteCodeFilePath; - return this; - } - - public String getByteCodeFilePath() { - return byteCodeFilePath; - } - public native long buildConfigInstance(); public native void releaseConfigInstance(long configId); + } From a88cd3705a4b49bea2f6af62dcaf910005c719d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 03:34:44 +0100 Subject: [PATCH 15/34] cmake format --- cpp/src/gandiva/precompiled/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index cbb38842718..12490008259 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -73,10 +73,7 @@ add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH} DEPENDS ${BC_FILES}) # write a cmake script to replace precompiled bitcode file's content into a .cc file -file( - WRITE - "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" - "\ +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" "\ file(READ \"${GANDIVA_PRECOMPILED_BC_PATH}\" HEXCONTENT HEX) string(REGEX REPLACE \"([0-9a-f][0-9a-f])\" From af78be26f980ea81a22d0f44fc0b8d26ed2eac2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 03:50:39 +0100 Subject: [PATCH 16/34] don't bundle irhelpers in the jar --- java/gandiva/pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index a0901530a61..0cc4682854d 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -29,7 +29,7 @@ 2.5.0 18.0 true - ../../cpp/debug/debug + ../cpp/build/debug @@ -132,12 +132,6 @@ **/libgandiva_helpers.* - - ${gandiva.cpp.build.dir}/../src/gandiva - - irhelpers.bc - - From 79abc0e88b690715eb7266f0cc003b7d08e05227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 13:36:46 +0100 Subject: [PATCH 17/34] mark .cc file as generated --- cpp/Dockerfile | 4 ++-- cpp/src/gandiva/CMakeLists.txt | 7 +++---- cpp/src/gandiva/jni/env_helper.h | 3 --- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cpp/Dockerfile b/cpp/Dockerfile index 4d177524e7d..b570bf679e7 100644 --- a/cpp/Dockerfile +++ b/cpp/Dockerfile @@ -43,12 +43,12 @@ RUN arrow/ci/docker_install_conda.sh && \ conda install -q -c conda-forge \ --file arrow/ci/conda_env_cpp.yml \ --file arrow/ci/conda_env_unix.yml \ - llvmdev $EXTRA_CONDA_PKGS && \ + $EXTRA_CONDA_PKGS && \ conda clean --all ENV CC=gcc \ CXX=g++ \ - ARROW_GANDIVA=ON \ + ARROW_GANDIVA=OFF \ ARROW_BUILD_TESTS=ON \ ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX \ ARROW_HOME=$CONDA_PREFIX \ diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 629fbcd2732..9d03ef13225 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -32,10 +32,9 @@ set(GANDIVA_PRECOMPILED_BC_PATH "${CMAKE_CURRENT_BINARY_DIR}/irhelpers.bc") set(GANDIVA_PRECOMPILED_CC_PATH "${CMAKE_CURRENT_BINARY_DIR}/precompiled_bitcode.cc") set(GANDIVA_PRECOMPILED_CC_IN_PATH "${CMAKE_CURRENT_SOURCE_DIR}/precompiled_bitcode.cc.in") -# add_arrow_lib will look for this not yet existing file, so write it -if(NOT EXISTS ${GANDIVA_PRECOMPILED_CC_PATH}) - configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH} ${GANDIVA_PRECOMPILED_CC_PATH}) -endif() + +# add_arrow_lib will look for this not yet existing file, so flag as generated +set_source_files_properties(${GANDIVA_PRECOMPILED_CC_PATH} PROPERTIES GENERATED TRUE) set(SRC_FILES annotator.cc diff --git a/cpp/src/gandiva/jni/env_helper.h b/cpp/src/gandiva/jni/env_helper.h index c1bfbaa82a5..1b56304f967 100644 --- a/cpp/src/gandiva/jni/env_helper.h +++ b/cpp/src/gandiva/jni/env_helper.h @@ -23,7 +23,4 @@ // class references extern jclass configuration_builder_class_; -// method references -extern jmethodID byte_code_accessor_method_id_; - #endif // ENV_HELPER_H From 09d829a409a1b3c95da7459c383b5176c965312f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 14:06:15 +0100 Subject: [PATCH 18/34] build wheels for a single python distribution at a time; adjust travis and crossbow scripts --- ci/travis_script_manylinux.sh | 63 ++++++---- dev/tasks/python-wheels/linux-test.sh | 1 + dev/tasks/python-wheels/travis.linux.yml | 8 +- python/manylinux1/build_arrow.sh | 143 +++++++++++------------ 4 files changed, 114 insertions(+), 101 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index a8afc86ab6d..837ea4e3e6e 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -20,15 +20,6 @@ set -ex -pushd python/manylinux1 -docker run --shm-size=2g --rm \ - -e PYARROW_PARALLEL=3 \ - -e PYTHON_VERSIONS=$PYTHON_VERSIONS \ - -v $PWD:/io \ - -v $PWD/../../:/arrow \ - quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 \ - /io/build_arrow.sh - # Testing for https://issues.apache.org/jira/browse/ARROW-2657 # These tests cannot be run inside of the docker container, since TensorFlow # does not run on manylinux1 @@ -36,18 +27,7 @@ docker run --shm-size=2g --rm \ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh -PYTHON_VERSION=3.6 -CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - -conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION -conda activate $CONDA_ENV_DIR - -pip install -q tensorflow -pip install "dist/`ls dist/ | grep cp36`" -popd - -# # Test optional dependencies and the presence of tensorflow -python -c " +cat << EOF > check_imports.py import sys import pyarrow import pyarrow.orc @@ -57,8 +37,41 @@ import tensorflow if sys.version_info.major > 2: import pyarrow.gandiva -" +EOF + +pushd python/manylinux1 -# Run pyarrow tests -pip install -r python/requirements-test.txt -pytest --pyargs pyarrow +for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do + IFS="," + set -- $PYTHON_TUPLE; + PYTHON_VERSION=$1 + UNICODE_WIDTH=$2 + + # build the wheels + docker run --shm-size=2g --rm \ + -e PYARROW_PARALLEL=3 \ + -e PYTHON_VERSION=$PYTHON_VERSION \ + -e UNICODE_WIDTH=$UNICODE_WIDTH \ + -v $PWD:/io \ + -v $PWD/../../:/arrow \ + quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 \ + /io/build_arrow.sh + + # create a testing conda environment + CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION + conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION + conda activate $CONDA_ENV_DIR + + # install the produced wheels + pip install -q tensorflow + pip install *.whl + + # Test optional dependencies and the presence of tensorflow + python check_imports.py + + # Install test dependencies and run pyarrow tests + pip install -r python/requirements-test.txt + pytest --pyargs pyarrow +done + +popd diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/linux-test.sh index fd660dc5754..dcd77eb56f5 100755 --- a/dev/tasks/python-wheels/linux-test.sh +++ b/dev/tasks/python-wheels/linux-test.sh @@ -30,6 +30,7 @@ import pyarrow import pyarrow.orc import pyarrow.parquet import pyarrow.plasma +import tensorflow if sys.version_info.major > 2: import pyarrow.gandiva diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml index b5cbc65bc7e..3d0ca8db03f 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.linux.yml @@ -30,7 +30,7 @@ env: - TRAVIS_TAG={{ task.tag }} before_script: - - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest + - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 script: - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow @@ -41,10 +41,12 @@ script: - pushd arrow/python/manylinux1 - docker run --shm-size=2g -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} - -e PYTHON_VERSIONS="{{ python_version }},{{ unicode_width }}" + -e PYTHON_VERSION="{{ python_version }}" + -e UNICODE_WIDTH="{{ unicode_width }}" -v $PWD:/io -v $PWD/../../:/arrow - quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh + quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 + /io/build_arrow.sh - popd # test on multiple distributions diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index ee4570e90a5..7ad28aefcc2 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -48,76 +48,73 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost - # Ensure the target directory exists mkdir -p /io/dist -for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do - IFS="," - set -- $PYTHON_TUPLE; - PYTHON=$1 - U_WIDTH=$2 - CPYTHON_PATH="$(cpython_path $PYTHON ${U_WIDTH})" - PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" - PIP="${CPYTHON_PATH}/bin/pip" - PATH="$PATH:${CPYTHON_PATH}" - - if [ $PYTHON != "2.7" ]; then - # Gandiva is not supported on Python 2.7 - export PYARROW_WITH_GANDIVA=1 - export BUILD_ARROW_GANDIVA=ON - else - export PYARROW_WITH_GANDIVA=0 - export BUILD_ARROW_GANDIVA=OFF - fi - - echo "=== (${PYTHON}) Building Arrow C++ libraries ===" - ARROW_BUILD_DIR=/tmp/build-PY${PYTHON}-${U_WIDTH} - mkdir -p "${ARROW_BUILD_DIR}" - pushd "${ARROW_BUILD_DIR}" - PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/arrow-dist \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BOOST_USE_SHARED=ON \ - -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ - -DARROW_JEMALLOC=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_PYTHON=ON \ - -DARROW_PARQUET=ON \ - -DPythonInterp_FIND_VERSION=${PYTHON} \ - -DARROW_PLASMA=ON \ - -DARROW_TENSORFLOW=ON \ - -DARROW_ORC=ON \ - -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ - -DARROW_GANDIVA_JAVA=OFF \ - -DBoost_NAMESPACE=arrow_boost \ - -DBOOST_ROOT=/arrow_boost_dist \ - -GNinja /arrow/cpp - ninja install - popd - - # Check that we don't expose any unwanted symbols - /io/scripts/check_arrow_visibility.sh - - echo "=== (${PYTHON}) Install the wheel build dependencies ===" - $PIP install -r requirements-wheel.txt - - # Clear output directory - rm -rf dist/ - echo "=== (${PYTHON}) Building wheel ===" - # Remove build directory to ensure CMake gets a clean run - rm -rf build/ - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext \ - --inplace \ - --bundle-arrow-cpp \ - --bundle-boost \ - --boost-namespace=arrow_boost - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel - PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist - - echo "=== (${PYTHON}) Tag the wheel with manylinux1 ===" - mkdir -p repaired_wheels/ - auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ - - # Testing happens outsite of the build to prevent issues like ARROW-4372 - mv repaired_wheels/*.whl /io/dist - mv dist/*.tar.gz /io/dist -done +# Must pass PYTHON_VERSION and UNICODE_WIDTH env variables +# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 + +CPYTHON_PATH="$(cpython_path $PYTHON ${UNICODE_WIDTH})" +PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" +PIP="${CPYTHON_PATH}/bin/pip" +PATH="$PATH:${CPYTHON_PATH}" + +if [ $PYTHON != "2.7" ]; then + # Gandiva is not supported on Python 2.7 + export PYARROW_WITH_GANDIVA=1 + export BUILD_ARROW_GANDIVA=ON +else + export PYARROW_WITH_GANDIVA=0 + export BUILD_ARROW_GANDIVA=OFF +fi + +echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" +ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH} +mkdir -p "${ARROW_BUILD_DIR}" +pushd "${ARROW_BUILD_DIR}" +PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/arrow-dist \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ + -DARROW_JEMALLOC=ON \ + -DARROW_RPATH_ORIGIN=ON \ + -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ + -DPythonInterp_FIND_VERSION=${PYTHON_VERSION} \ + -DARROW_PLASMA=ON \ + -DARROW_TENSORFLOW=ON \ + -DARROW_ORC=ON \ + -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ + -DARROW_GANDIVA_JAVA=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -GNinja /arrow/cpp +ninja install +popd + +# Check that we don't expose any unwanted symbols +/io/scripts/check_arrow_visibility.sh + +echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" +$PIP install -r requirements-wheel.txt + +# Clear output directory +rm -rf dist/ +echo "=== (${PYTHON_VERSION}) Building wheel ===" +# Remove build directory to ensure CMake gets a clean run +rm -rf build/ +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext \ + --inplace \ + --bundle-arrow-cpp \ + --bundle-boost \ + --boost-namespace=arrow_boost +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist + +echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux1 ===" +mkdir -p repaired_wheels/ +auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ + +# Testing happens outsite of the build to prevent issues like ARROW-4372 +mv repaired_wheels/*.whl /io/dist +mv dist/*.tar.gz /io/dist From 5d75adb40a14948f42edd9462fbc98cc490d1464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 14:18:48 +0100 Subject: [PATCH 19/34] initialize jni loader --- .../main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java index 95e3cadbd99..b6174d723d0 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java @@ -119,6 +119,7 @@ static long getDefaultConfiguration() throws GandivaException { if (defaultConfiguration == 0L) { synchronized (ConfigurationBuilder.class) { if (defaultConfiguration == 0L) { + JniLoader.getInstance(); // setup defaultConfiguration = new ConfigurationBuilder() .buildConfigInstance(); } From 0d29b3115418a8b1f09583125dc962b69d867916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 14:43:44 +0100 Subject: [PATCH 20/34] remove not existing resources from gandiva's pom --- java/gandiva/pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 0cc4682854d..3469c9f0030 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -29,7 +29,7 @@ 2.5.0 18.0 true - ../cpp/build/debug + ../../cpp/build/debug @@ -126,12 +126,6 @@ **/libgandiva_jni.* - - ${gandiva.cpp.build.dir} - - **/libgandiva_helpers.* - - From d3cb058b1d21230c16425ccf8a29ade910f216d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 15:40:11 +0100 Subject: [PATCH 21/34] fix wheel building script --- python/manylinux1/build_arrow.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 7ad28aefcc2..09c29f82ffb 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -51,12 +51,12 @@ mkdir -p /io/dist # Must pass PYTHON_VERSION and UNICODE_WIDTH env variables # possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 -CPYTHON_PATH="$(cpython_path $PYTHON ${UNICODE_WIDTH})" +CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" PIP="${CPYTHON_PATH}/bin/pip" -PATH="$PATH:${CPYTHON_PATH}" +PATH="${PATH}:${CPYTHON_PATH}" -if [ $PYTHON != "2.7" ]; then +if [ ${PYTHON_VERSION} != "2.7" ]; then # Gandiva is not supported on Python 2.7 export PYARROW_WITH_GANDIVA=1 export BUILD_ARROW_GANDIVA=ON @@ -69,7 +69,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH} mkdir -p "${ARROW_BUILD_DIR}" pushd "${ARROW_BUILD_DIR}" -PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release \ +PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/arrow-dist \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_BUILD_TESTS=OFF \ From 54fc653ba989fd04302dcc88dbb81ea99ed28df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 16:14:58 +0100 Subject: [PATCH 22/34] don't export --- cpp/src/gandiva/engine.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index c4c19e74f72..8fbf2f58397 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -66,7 +66,6 @@ namespace gandiva { -GANDIVA_EXPORT extern const char kPrecompiledBitcode[]; extern const size_t kPrecompiledBitcodeSize; From 0bb07a7e0aff472e8bcfeb4d813dde0553bc2163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 17:14:43 +0100 Subject: [PATCH 23/34] better bash split --- ci/travis_script_manylinux.sh | 5 +---- python/manylinux1/build_arrow.sh | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 837ea4e3e6e..651a480f27e 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -42,10 +42,7 @@ EOF pushd python/manylinux1 for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do - IFS="," - set -- $PYTHON_TUPLE; - PYTHON_VERSION=$1 - UNICODE_WIDTH=$2 + IFS="," read PYTHON_VERSION UNICODE_WIDTH <<< $PYTHON_TUPLE # build the wheels docker run --shm-size=2g --rm \ diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 09c29f82ffb..70992974300 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -23,9 +23,6 @@ # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) -# Build different python versions with various unicode widths -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16 3.7,16}" - source /multibuild/manylinux_utils.sh # Quit on failure From 18c548835beaecc09a33d395eae2341fd1516ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 18:15:53 +0100 Subject: [PATCH 24/34] install wheel from dist dir --- ci/travis_script_manylinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 651a480f27e..6536a611f8f 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -61,7 +61,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do # install the produced wheels pip install -q tensorflow - pip install *.whl + pip install dist/*.whl # Test optional dependencies and the presence of tensorflow python check_imports.py From 7c88d61b7b7aad6ce36d3918a180f99217039c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 20:30:53 +0100 Subject: [PATCH 25/34] only build python 3.6 wheel --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7bfe10ac059..fffd6a584b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -226,7 +226,7 @@ matrix: - name: "[manylinux1] Python" language: cpp env: - - PYTHON_VERSIONS=2.7,16 3.6,16 + - PYTHON_VERSIONS=3.6,16 before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: From 3e4ec2a2d176eabc574daf196c45f99473cd7fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 19 Feb 2019 21:27:20 +0100 Subject: [PATCH 26/34] unterminated llvm:MemoryBuffer; fix check_import.py path --- ci/travis_script_manylinux.sh | 4 ++-- cpp/src/gandiva/engine.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 6536a611f8f..fbc3f06f3a2 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -27,6 +27,8 @@ set -ex source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh +pushd python/manylinux1 + cat << EOF > check_imports.py import sys import pyarrow @@ -39,8 +41,6 @@ if sys.version_info.major > 2: import pyarrow.gandiva EOF -pushd python/manylinux1 - for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do IFS="," read PYTHON_VERSION UNICODE_WIDTH <<< $PYTHON_TUPLE diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 8fbf2f58397..22805bb80f6 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -134,7 +134,7 @@ Status Engine::LoadPreCompiledIR() { /// Read from file into memory buffer. llvm::ErrorOr> buffer_or_error = - llvm::MemoryBuffer::getMemBuffer(bitcode); + llvm::MemoryBuffer::getMemBuffer(bitcode, "precompiled", false); ARROW_RETURN_IF(!buffer_or_error, Status::CodeGenError("Could not load module from IR: ", From 2372f3d7c5bf1e68ace3530d7023d4075a941bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 00:06:14 +0100 Subject: [PATCH 27/34] fix requirements path; disable other CI tests --- .travis.yml | 664 +++++++++++++++++----------------- ci/travis_script_manylinux.sh | 2 +- 2 files changed, 333 insertions(+), 333 deletions(-) diff --git a/.travis.yml b/.travis.yml index fffd6a584b4..a62075853d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,345 +46,345 @@ before_install: matrix: fast_finish: true include: - - name: "Lint C++, Python, R" - os: linux - language: python - python: "3.6" - before_script: - # Always run RAT checks, in case another build in matrix breaks RAT - - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - - name: "C++ unit tests w/ Valgrind, clang 7.0" - language: cpp - os: linux - env: - - ARROW_TRAVIS_VALGRIND=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - CC="clang-7" - - CXX="clang++-7" - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # Separating Valgrind and C++ coverage makes individual jobs shorter - - name: "C++ unit tests w/ gcc 5.4, coverage" - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - - ARROW_TRAVIS_COVERAGE=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 - - name: "C++ unit tests w/ gcc 4.8, trusty" - dist: trusty - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - before_install: - - ulimit -c unlimited -S - - | - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq - fi - - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage" - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - # Valgrind is needed for the Plasma store tests - - ARROW_TRAVIS_VALGRIND=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_COVERAGE=1 - - ARROW_TRAVIS_PYTHON_DOCS=1 - - ARROW_TRAVIS_PYTHON_JVM=1 - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - # TODO(wesm): Run the benchmarks outside of Travis - # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 - before_script: - - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 - - ARROW_TRAVIS_PYTHON_GANDIVA=1 - # Only run Plasma tests with valgrind in one of the Python builds because - # they are slow - - export PLASMA_VALGRIND=0 - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 - - export PLASMA_VALGRIND=1 - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh - - name: "[OS X] C++ w/ XCode 8.3" - compiler: clang - language: cpp - osx_image: xcode8.3 - os: osx - cache: - addons: - env: - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_TRAVIS_VERBOSE=0 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib - # which can get loaded before the toolchain Boost libraries. These seem to - # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own - # Boost and statically link to get around the issue until this can be - # investigated further - - ARROW_TRAVIS_VENDORED_BOOST=1 - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh - - name: "[OS X] Python w/ XCode 7.3" - compiler: clang - language: cpp - osx_image: xcode7.3 - os: osx - cache: - addons: - env: - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - MACOSX_DEPLOYMENT_TARGET="10.9" - before_script: - script: - - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 + # - name: "Lint C++, Python, R" + # os: linux + # language: python + # python: "3.6" + # before_script: + # # Always run RAT checks, in case another build in matrix breaks RAT + # - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_lint.sh + # - name: "C++ unit tests w/ Valgrind, clang 7.0" + # language: cpp + # os: linux + # env: + # - ARROW_TRAVIS_VALGRIND=1 + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_TRAVIS_FLIGHT=1 + # - ARROW_TRAVIS_PLASMA=1 + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # - CC="clang-7" + # - CXX="clang++-7" + # before_script: + # - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # # If either C++ or Python changed, we must install the C++ libraries + # - git submodule update --init + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + # # Separating Valgrind and C++ coverage makes individual jobs shorter + # - name: "C++ unit tests w/ gcc 5.4, coverage" + # compiler: gcc + # language: cpp + # os: linux + # jdk: openjdk8 + # env: + # - ARROW_TRAVIS_COVERAGE=1 + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_TRAVIS_FLIGHT=1 + # - ARROW_TRAVIS_PLASMA=1 + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_GANDIVA_JAVA=1 + # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # before_script: + # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # # If either C++ or Python changed, we must install the C++ libraries + # - git submodule update --init + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 + # - name: "C++ unit tests w/ gcc 4.8, trusty" + # dist: trusty + # compiler: gcc + # language: cpp + # os: linux + # jdk: openjdk8 + # env: + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_TRAVIS_PLASMA=1 + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_GANDIVA_JAVA=1 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # before_install: + # - ulimit -c unlimited -S + # - | + # if [ $TRAVIS_OS_NAME == "linux" ]; then + # sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" + # sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + # sudo apt-get update -qq + # fi + # - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` + # before_script: + # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # # If either C++ or Python changed, we must install the C++ libraries + # - git submodule update --init + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 + # - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage" + # compiler: gcc + # language: cpp + # os: linux + # jdk: openjdk8 + # env: + # # Valgrind is needed for the Plasma store tests + # - ARROW_TRAVIS_VALGRIND=1 + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_TRAVIS_COVERAGE=1 + # - ARROW_TRAVIS_PYTHON_DOCS=1 + # - ARROW_TRAVIS_PYTHON_JVM=1 + # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + # # TODO(wesm): Run the benchmarks outside of Travis + # # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 + # before_script: + # - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 + # - ARROW_TRAVIS_PYTHON_GANDIVA=1 + # # Only run Plasma tests with valgrind in one of the Python builds because + # # they are slow + # - export PLASMA_VALGRIND=0 + # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 + # - export PLASMA_VALGRIND=1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh + # - name: "[OS X] C++ w/ XCode 8.3" + # compiler: clang + # language: cpp + # osx_image: xcode8.3 + # os: osx + # cache: + # addons: + # env: + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_TRAVIS_PLASMA=1 + # - ARROW_TRAVIS_FLIGHT=1 + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_GANDIVA_JAVA=1 + # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + # - ARROW_TRAVIS_VERBOSE=0 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib + # # which can get loaded before the toolchain Boost libraries. These seem to + # # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own + # # Boost and statically link to get around the issue until this can be + # # investigated further + # - ARROW_TRAVIS_VENDORED_BOOST=1 + # before_script: + # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # # If either C++ or Python changed, we must install the C++ libraries + # - git submodule update --init + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh + # - name: "[OS X] Python w/ XCode 7.3" + # compiler: clang + # language: cpp + # osx_image: xcode7.3 + # os: osx + # cache: + # addons: + # env: + # - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + # - MACOSX_DEPLOYMENT_TARGET="10.9" + # before_script: + # script: + # - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 + # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" language: cpp env: - - PYTHON_VERSIONS=3.6,16 + - PYTHON_VERSIONS="3.6,16 2.7,16" before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi - - name: "Java w/ OpenJDK 8" - language: java - os: linux - jdk: openjdk8 - before_script: - - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh - - name: "Java w/ OpenJDK 9" - language: java - os: linux - jdk: openjdk9 - before_script: - - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - - name: "Java w/ OpenJDK 11" - language: java - os: linux - jdk: openjdk11 - before_script: - - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - - name: "Integration w/ OpenJDK 8, clang 7" - language: java - os: linux - env: ARROW_TEST_GROUP=integration - jdk: openjdk8 - env: - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1 - - CC="clang-7" - - CXX="clang++-7" - before_script: - - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - nvm install 11.6 - - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh - - name: "NodeJS" - language: node_js - os: linux - node_js: - - '11.6' - before_script: - - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - - name: "C++ & GLib & Ruby w/ gcc 5.4" - compiler: gcc - language: cpp - os: linux - env: - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_USE_VENDORED_BOOST=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PLASMA=1 - before_script: - - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - script: - - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - - name: "[OS X] C++ & GLib & Ruby w/ XCode 8.3 & homebrew" - compiler: clang - osx_image: xcode8.3 - os: osx - env: - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PLASMA=1 - cache: - addons: - before_script: - - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew - - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - script: - - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - - name: Rust - language: rust - cache: cargo - addons: - apt: - packages: - - libcurl4-openssl-dev - - libelf-dev - - libdw-dev - - binutils-dev - - cmake - before_script: - - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh - script: - - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh - after_success: - - pushd ${TRAVIS_BUILD_DIR}/rust - # Run coverage for codecov.io - - mkdir -p target/kcov - - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly cargo coverage --verbose - - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" - - name: Go - language: go - go_import_path: github.com/apache/arrow - os: linux - go: - - 1.11.x - before_script: - - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh - after_success: - - pushd ${TRAVIS_BUILD_DIR}/go/arrow - - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" - - name: R - language: r - cache: packages - latex: false - dist: trusty - env: - - ARROW_TRAVIS_PARQUET=1 - before_install: - # Have to copy-paste this here because of how R's build steps work - - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi - - | - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq - fi - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib - - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig - - pushd ${TRAVIS_BUILD_DIR}/r + # - name: "Java w/ OpenJDK 8" + # language: java + # os: linux + # jdk: openjdk8 + # before_script: + # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + # - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh + # - name: "Java w/ OpenJDK 9" + # language: java + # os: linux + # jdk: openjdk9 + # before_script: + # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + # - name: "Java w/ OpenJDK 11" + # language: java + # os: linux + # jdk: openjdk11 + # before_script: + # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + # - name: "Integration w/ OpenJDK 8, clang 7" + # language: java + # os: linux + # env: ARROW_TEST_GROUP=integration + # jdk: openjdk8 + # env: + # - ARROW_TRAVIS_PLASMA=1 + # - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1 + # - CC="clang-7" + # - CXX="clang++-7" + # before_script: + # - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # - nvm install 11.6 + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh + # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh + # - name: "NodeJS" + # language: node_js + # os: linux + # node_js: + # - '11.6' + # before_script: + # - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + # - name: "C++ & GLib & Ruby w/ gcc 5.4" + # compiler: gcc + # language: cpp + # os: linux + # env: + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_USE_VENDORED_BOOST=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_PLASMA=1 + # before_script: + # - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh + # script: + # - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi + # - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + # - name: "[OS X] C++ & GLib & Ruby w/ XCode 8.3 & homebrew" + # compiler: clang + # osx_image: xcode8.3 + # os: osx + # env: + # - ARROW_TRAVIS_ORC=1 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_PARQUET=1 + # - ARROW_TRAVIS_PLASMA=1 + # cache: + # addons: + # before_script: + # - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh + # script: + # - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi + # - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + # - name: Rust + # language: rust + # cache: cargo + # addons: + # apt: + # packages: + # - libcurl4-openssl-dev + # - libelf-dev + # - libdw-dev + # - binutils-dev + # - cmake + # before_script: + # - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh + # script: + # - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh + # after_success: + # - pushd ${TRAVIS_BUILD_DIR}/rust + # # Run coverage for codecov.io + # - mkdir -p target/kcov + # - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly cargo coverage --verbose + # - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + # - name: Go + # language: go + # go_import_path: github.com/apache/arrow + # os: linux + # go: + # - 1.11.x + # before_script: + # - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh + # after_success: + # - pushd ${TRAVIS_BUILD_DIR}/go/arrow + # - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + # - name: R + # language: r + # cache: packages + # latex: false + # dist: trusty + # env: + # - ARROW_TRAVIS_PARQUET=1 + # before_install: + # # Have to copy-paste this here because of how R's build steps work + # - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` + # - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi + # - | + # if [ $TRAVIS_OS_NAME == "linux" ]; then + # sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" + # sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + # sudo apt-get update -qq + # fi + # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library + # - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib + # - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig + # - pushd ${TRAVIS_BUILD_DIR}/r after_failure: diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index fbc3f06f3a2..b4d33465a74 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -67,7 +67,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do python check_imports.py # Install test dependencies and run pyarrow tests - pip install -r python/requirements-test.txt + pip install -r ../requirements-test.txt pytest --pyargs pyarrow done From 71233c7da804db07b11101a6c96011adc08193d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 00:57:12 +0100 Subject: [PATCH 28/34] test 2.7,16 wheel --- .travis.yml | 2 +- ci/travis_script_manylinux.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a62075853d3..af21a2cebf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -226,7 +226,7 @@ matrix: - name: "[manylinux1] Python" language: cpp env: - - PYTHON_VERSIONS="3.6,16 2.7,16" + - PYTHON_VERSIONS="2.7,16" before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index b4d33465a74..0a607263ac3 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -60,7 +60,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do conda activate $CONDA_ENV_DIR # install the produced wheels - pip install -q tensorflow + pip install tensorflow pip install dist/*.whl # Test optional dependencies and the presence of tensorflow From b3994962dd33cdbae549ebcd347b7aae7e737ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 01:17:22 +0100 Subject: [PATCH 29/34] test py27mu and py36m wheels --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index af21a2cebf2..0a521afebef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -226,7 +226,7 @@ matrix: - name: "[manylinux1] Python" language: cpp env: - - PYTHON_VERSIONS="2.7,16" + - PYTHON_VERSIONS="2.7,32 3.6,16" before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: From 1aa19f184068c7d310d1e4acab812c7e06c93ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 01:20:07 +0100 Subject: [PATCH 30/34] reenable travis builds --- .travis.yml | 662 ++++++++++++++++++++++++++-------------------------- 1 file changed, 331 insertions(+), 331 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a521afebef..7458ab6ed87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,183 +46,183 @@ before_install: matrix: fast_finish: true include: - # - name: "Lint C++, Python, R" - # os: linux - # language: python - # python: "3.6" - # before_script: - # # Always run RAT checks, in case another build in matrix breaks RAT - # - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - # - name: "C++ unit tests w/ Valgrind, clang 7.0" - # language: cpp - # os: linux - # env: - # - ARROW_TRAVIS_VALGRIND=1 - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_TRAVIS_FLIGHT=1 - # - ARROW_TRAVIS_PLASMA=1 - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # - CC="clang-7" - # - CXX="clang++-7" - # before_script: - # - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # # If either C++ or Python changed, we must install the C++ libraries - # - git submodule update --init - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # # Separating Valgrind and C++ coverage makes individual jobs shorter - # - name: "C++ unit tests w/ gcc 5.4, coverage" - # compiler: gcc - # language: cpp - # os: linux - # jdk: openjdk8 - # env: - # - ARROW_TRAVIS_COVERAGE=1 - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_TRAVIS_FLIGHT=1 - # - ARROW_TRAVIS_PLASMA=1 - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_GANDIVA_JAVA=1 - # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # before_script: - # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # # If either C++ or Python changed, we must install the C++ libraries - # - git submodule update --init - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 - # - name: "C++ unit tests w/ gcc 4.8, trusty" - # dist: trusty - # compiler: gcc - # language: cpp - # os: linux - # jdk: openjdk8 - # env: - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_TRAVIS_PLASMA=1 - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_GANDIVA_JAVA=1 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # before_install: - # - ulimit -c unlimited -S - # - | - # if [ $TRAVIS_OS_NAME == "linux" ]; then - # sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - # sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - # sudo apt-get update -qq - # fi - # - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - # before_script: - # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # # If either C++ or Python changed, we must install the C++ libraries - # - git submodule update --init - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - # - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage" - # compiler: gcc - # language: cpp - # os: linux - # jdk: openjdk8 - # env: - # # Valgrind is needed for the Plasma store tests - # - ARROW_TRAVIS_VALGRIND=1 - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_TRAVIS_COVERAGE=1 - # - ARROW_TRAVIS_PYTHON_DOCS=1 - # - ARROW_TRAVIS_PYTHON_JVM=1 - # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - # # TODO(wesm): Run the benchmarks outside of Travis - # # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 - # before_script: - # - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 - # - ARROW_TRAVIS_PYTHON_GANDIVA=1 - # # Only run Plasma tests with valgrind in one of the Python builds because - # # they are slow - # - export PLASMA_VALGRIND=0 - # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 - # - export PLASMA_VALGRIND=1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh - # - name: "[OS X] C++ w/ XCode 8.3" - # compiler: clang - # language: cpp - # osx_image: xcode8.3 - # os: osx - # cache: - # addons: - # env: - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_TRAVIS_PLASMA=1 - # - ARROW_TRAVIS_FLIGHT=1 - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_GANDIVA_JAVA=1 - # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - # - ARROW_TRAVIS_VERBOSE=0 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib - # # which can get loaded before the toolchain Boost libraries. These seem to - # # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own - # # Boost and statically link to get around the issue until this can be - # # investigated further - # - ARROW_TRAVIS_VENDORED_BOOST=1 - # before_script: - # - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # # If either C++ or Python changed, we must install the C++ libraries - # - git submodule update --init - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh - # - name: "[OS X] Python w/ XCode 7.3" - # compiler: clang - # language: cpp - # osx_image: xcode7.3 - # os: osx - # cache: - # addons: - # env: - # - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - # - MACOSX_DEPLOYMENT_TARGET="10.9" - # before_script: - # script: - # - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 - # - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 + - name: "Lint C++, Python, R" + os: linux + language: python + python: "3.6" + before_script: + # Always run RAT checks, in case another build in matrix breaks RAT + - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh + - name: "C++ unit tests w/ Valgrind, clang 7.0" + language: cpp + os: linux + env: + - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_FLIGHT=1 + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + - CC="clang-7" + - CXX="clang++-7" + before_script: + - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # If either C++ or Python changed, we must install the C++ libraries + - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + # Separating Valgrind and C++ coverage makes individual jobs shorter + - name: "C++ unit tests w/ gcc 5.4, coverage" + compiler: gcc + language: cpp + os: linux + jdk: openjdk8 + env: + - ARROW_TRAVIS_COVERAGE=1 + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_FLIGHT=1 + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_GANDIVA_JAVA=1 + - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + before_script: + - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # If either C++ or Python changed, we must install the C++ libraries + - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 + - name: "C++ unit tests w/ gcc 4.8, trusty" + dist: trusty + compiler: gcc + language: cpp + os: linux + jdk: openjdk8 + env: + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_GANDIVA_JAVA=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + before_install: + - ulimit -c unlimited -S + - | + if [ $TRAVIS_OS_NAME == "linux" ]; then + sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -qq + fi + - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` + before_script: + - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + # If either C++ or Python changed, we must install the C++ libraries + - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 + - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage" + compiler: gcc + language: cpp + os: linux + jdk: openjdk8 + env: + # Valgrind is needed for the Plasma store tests + - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_COVERAGE=1 + - ARROW_TRAVIS_PYTHON_DOCS=1 + - ARROW_TRAVIS_PYTHON_JVM=1 + - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + # TODO(wesm): Run the benchmarks outside of Travis + # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 + before_script: + - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 + - ARROW_TRAVIS_PYTHON_GANDIVA=1 + # Only run Plasma tests with valgrind in one of the Python builds because + # they are slow + - export PLASMA_VALGRIND=0 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 + - export PLASMA_VALGRIND=1 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh + - name: "[OS X] C++ w/ XCode 8.3" + compiler: clang + language: cpp + osx_image: xcode8.3 + os: osx + cache: + addons: + env: + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_FLIGHT=1 + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_GANDIVA_JAVA=1 + - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + - ARROW_TRAVIS_VERBOSE=0 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib + # which can get loaded before the toolchain Boost libraries. These seem to + # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own + # Boost and statically link to get around the issue until this can be + # investigated further + - ARROW_TRAVIS_VENDORED_BOOST=1 + before_script: + - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + # If either C++ or Python changed, we must install the C++ libraries + - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh + - name: "[OS X] Python w/ XCode 7.3" + compiler: clang + language: cpp + osx_image: xcode7.3 + os: osx + cache: + addons: + env: + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN + - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + - MACOSX_DEPLOYMENT_TARGET="10.9" + before_script: + script: + - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 + - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" language: cpp env: @@ -231,160 +231,160 @@ matrix: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi - # - name: "Java w/ OpenJDK 8" - # language: java - # os: linux - # jdk: openjdk8 - # before_script: - # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - # - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh - # - name: "Java w/ OpenJDK 9" - # language: java - # os: linux - # jdk: openjdk9 - # before_script: - # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - # - name: "Java w/ OpenJDK 11" - # language: java - # os: linux - # jdk: openjdk11 - # before_script: - # - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - # - name: "Integration w/ OpenJDK 8, clang 7" - # language: java - # os: linux - # env: ARROW_TEST_GROUP=integration - # jdk: openjdk8 - # env: - # - ARROW_TRAVIS_PLASMA=1 - # - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1 - # - CC="clang-7" - # - CXX="clang++-7" - # before_script: - # - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # - nvm install 11.6 - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh - # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh - # - name: "NodeJS" - # language: node_js - # os: linux - # node_js: - # - '11.6' - # before_script: - # - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - # - name: "C++ & GLib & Ruby w/ gcc 5.4" - # compiler: gcc - # language: cpp - # os: linux - # env: - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_USE_VENDORED_BOOST=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_PLASMA=1 - # before_script: - # - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - # script: - # - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - # - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - # - name: "[OS X] C++ & GLib & Ruby w/ XCode 8.3 & homebrew" - # compiler: clang - # osx_image: xcode8.3 - # os: osx - # env: - # - ARROW_TRAVIS_ORC=1 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_PARQUET=1 - # - ARROW_TRAVIS_PLASMA=1 - # cache: - # addons: - # before_script: - # - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - # script: - # - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - # - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - # - name: Rust - # language: rust - # cache: cargo - # addons: - # apt: - # packages: - # - libcurl4-openssl-dev - # - libelf-dev - # - libdw-dev - # - binutils-dev - # - cmake - # before_script: - # - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh - # script: - # - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh - # after_success: - # - pushd ${TRAVIS_BUILD_DIR}/rust - # # Run coverage for codecov.io - # - mkdir -p target/kcov - # - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly cargo coverage --verbose - # - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" - # - name: Go - # language: go - # go_import_path: github.com/apache/arrow - # os: linux - # go: - # - 1.11.x - # before_script: - # - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh - # after_success: - # - pushd ${TRAVIS_BUILD_DIR}/go/arrow - # - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" - # - name: R - # language: r - # cache: packages - # latex: false - # dist: trusty - # env: - # - ARROW_TRAVIS_PARQUET=1 - # before_install: - # # Have to copy-paste this here because of how R's build steps work - # - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - # - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi - # - | - # if [ $TRAVIS_OS_NAME == "linux" ]; then - # sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - # sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - # sudo apt-get update -qq - # fi - # - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - # - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib - # - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig - # - pushd ${TRAVIS_BUILD_DIR}/r + - name: "Java w/ OpenJDK 8" + language: java + os: linux + jdk: openjdk8 + before_script: + - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh + - name: "Java w/ OpenJDK 9" + language: java + os: linux + jdk: openjdk9 + before_script: + - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + - name: "Java w/ OpenJDK 11" + language: java + os: linux + jdk: openjdk11 + before_script: + - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh + - name: "Integration w/ OpenJDK 8, clang 7" + language: java + os: linux + env: ARROW_TEST_GROUP=integration + jdk: openjdk8 + env: + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1 + - CC="clang-7" + - CXX="clang++-7" + before_script: + - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - nvm install 11.6 + - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh + - name: "NodeJS" + language: node_js + os: linux + node_js: + - '11.6' + before_script: + - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + - name: "C++ & GLib & Ruby w/ gcc 5.4" + compiler: gcc + language: cpp + os: linux + env: + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_USE_VENDORED_BOOST=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_PLASMA=1 + before_script: + - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library + - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh + script: + - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi + - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + - name: "[OS X] C++ & GLib & Ruby w/ XCode 8.3 & homebrew" + compiler: clang + osx_image: xcode8.3 + os: osx + env: + - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_PLASMA=1 + cache: + addons: + before_script: + - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew + - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh + script: + - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi + - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + - name: Rust + language: rust + cache: cargo + addons: + apt: + packages: + - libcurl4-openssl-dev + - libelf-dev + - libdw-dev + - binutils-dev + - cmake + before_script: + - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh + script: + - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh + after_success: + - pushd ${TRAVIS_BUILD_DIR}/rust + # Run coverage for codecov.io + - mkdir -p target/kcov + - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly cargo coverage --verbose + - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + - name: Go + language: go + go_import_path: github.com/apache/arrow + os: linux + go: + - 1.11.x + before_script: + - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh + after_success: + - pushd ${TRAVIS_BUILD_DIR}/go/arrow + - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + - name: R + language: r + cache: packages + latex: false + dist: trusty + env: + - ARROW_TRAVIS_PARQUET=1 + before_install: + # Have to copy-paste this here because of how R's build steps work + - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` + - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi + - | + if [ $TRAVIS_OS_NAME == "linux" ]; then + sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -qq + fi + - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library + - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib + - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig + - pushd ${TRAVIS_BUILD_DIR}/r after_failure: From d5531d951e8dcfab4d11cf6b4098913d0253dcd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 13:42:39 +0100 Subject: [PATCH 31/34] test imports inside the wheel container --- ci/travis_script_manylinux.sh | 3 +++ python/manylinux1/build_arrow.sh | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 0a607263ac3..22afd76bec5 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -44,6 +44,9 @@ EOF for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do IFS="," read PYTHON_VERSION UNICODE_WIDTH <<< $PYTHON_TUPLE + # cleanup the artifact directory + rm -rf dist + # build the wheels docker run --shm-size=2g --rm \ -e PYARROW_PARALLEL=3 \ diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 70992974300..cc28819712d 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -112,6 +112,22 @@ echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux1 ===" mkdir -p repaired_wheels/ auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ -# Testing happens outsite of the build to prevent issues like ARROW-4372 -mv repaired_wheels/*.whl /io/dist +# Install the built wheels +pip install repaired_wheels/*.whl + +# Test that the modules are importable +python -c " +import sys +import pyarrow +import pyarrow.orc +import pyarrow.parquet +import pyarrow.plasma + +if sys.version_info.major > 2: + import pyarrow.gandiva +" + +# More thorough testing happens outsite of the build to prevent +# packaging issues like ARROW-4372 mv dist/*.tar.gz /io/dist +mv repaired_wheels/*.whl /io/dist From fd5e3feadfda0bbd43c63bcaa6f4c1c9bf8b90b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 13:43:35 +0100 Subject: [PATCH 32/34] use latest docker image tag --- ci/travis_script_manylinux.sh | 2 +- dev/tasks/python-wheels/travis.linux.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 22afd76bec5..4559be86755 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -54,7 +54,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do -e UNICODE_WIDTH=$UNICODE_WIDTH \ -v $PWD:/io \ -v $PWD/../../:/arrow \ - quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 \ + quay.io/xhochy/arrow_manylinux1_x86_64_base:latest \ /io/build_arrow.sh # create a testing conda environment diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml index 3d0ca8db03f..240c523e1cf 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.linux.yml @@ -30,7 +30,7 @@ env: - TRAVIS_TAG={{ task.tag }} before_script: - - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1 + - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest script: - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow From c573a564cff5c86f306f9d55c3ec80a4f666d3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 20 Feb 2019 15:32:50 +0100 Subject: [PATCH 33/34] use env variables insude the container --- python/manylinux1/build_arrow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index cc28819712d..e0475bb6c48 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -113,10 +113,10 @@ mkdir -p repaired_wheels/ auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/ # Install the built wheels -pip install repaired_wheels/*.whl +$PIP install repaired_wheels/*.whl # Test that the modules are importable -python -c " +$PYTHON_INTERPRETER -c " import sys import pyarrow import pyarrow.orc From 3b1da30e972fb6f2b441d10862ddcba7b20ac8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 21 Feb 2019 10:55:53 +0100 Subject: [PATCH 34/34] use sudo --- ci/travis_script_manylinux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 4559be86755..fa02c82e1bf 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -44,8 +44,8 @@ EOF for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do IFS="," read PYTHON_VERSION UNICODE_WIDTH <<< $PYTHON_TUPLE - # cleanup the artifact directory - rm -rf dist + # cleanup the artifact directory, docker writes it as root + sudo rm -rf dist # build the wheels docker run --shm-size=2g --rm \