From c5c4a8c8835b937a708180400b6bc25e51782cfd Mon Sep 17 00:00:00 2001 From: Max Risuhin Date: Sat, 29 Apr 2017 19:33:42 +0300 Subject: [PATCH] PARQUET-679: Local Windows build and Appveyor support --- CMakeLists.txt | 100 +++---- LICENSE.txt | 11 + README.md | 6 + appveyor.yml | 36 +++ benchmarks/decode_benchmark.cc | 8 +- ci/msvc-build.bat | 27 ++ cmake_modules/BuildUtils.cmake | 112 ++++++++ cmake_modules/CompilerInfo.cmake | 5 +- cmake_modules/SnappyCMakeLists.txt | 85 ++++++ cmake_modules/SnappyConfig.h | 36 +++ cmake_modules/ThirdpartyToolchain.cmake | 338 +++++++++++++++++------ docs/Windows.md | 76 +++++ src/parquet/arrow/CMakeLists.txt | 71 ++--- src/parquet/column/levels.h | 4 +- src/parquet/compression.h | 6 +- src/parquet/encoding-internal.h | 4 +- src/parquet/file/reader-internal.h | 7 +- src/parquet/parquet.thrift | 2 + src/parquet/util/bit-util.h | 18 +- src/parquet/util/compiler-util.h | 5 + src/parquet/util/cpu-info.cc | 3 + src/parquet/util/cpu-info.h | 4 +- src/parquet/util/memory.h | 13 +- src/parquet/util/rle-test.cc | 19 +- src/parquet/util/stopwatch.h | 2 + src/parquet/util/visibility.h | 1 + src/parquet/util/windows_compatibility.h | 37 +++ tools/parquet-scan.cc | 2 +- 28 files changed, 795 insertions(+), 243 deletions(-) create mode 100644 appveyor.yml create mode 100644 ci/msvc-build.bat create mode 100644 cmake_modules/BuildUtils.cmake create mode 100644 cmake_modules/SnappyCMakeLists.txt create mode 100644 cmake_modules/SnappyConfig.h create mode 100644 docs/Windows.md create mode 100644 src/parquet/util/windows_compatibility.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b85d96a..46d43944 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,13 +111,15 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build our own zlib (some libz.a aren't configured for static linking)" ON) option(PARQUET_RPATH_ORIGIN - "Build Parquet libraries with RATH set to \$ORIGIN" + "Build Parquet libraries with RPATH set to \$ORIGIN" OFF) option(PARQUET_MINIMAL_DEPENDENCY "Depend only on Thirdparty headers to build libparquet. Always OFF if building binaries" OFF) endif() +include(BuildUtils) + if (PARQUET_BUILD_TESTS OR PARQUET_BUILD_EXECUTABLES OR PARQUET_BUILD_BENCHMARKS) set(PARQUET_BUILD_STATIC ON) set(PARQUET_MINIMAL_DEPENDENCY OFF) @@ -283,6 +285,8 @@ function(ADD_PARQUET_TEST REL_TEST_NAME) COMPILE_FLAGS " -DPARQUET_VALGRIND") add_test(${TEST_NAME} valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH}) + elseif(MSVC) + add_test(${TEST_NAME} ${TEST_PATH}) else() add_test(${TEST_NAME} ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH}) @@ -328,8 +332,12 @@ endif() include(ThirdpartyToolchain) # Thrift requires these definitions for some types that we use -add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DHAVE_NETDB_H) -add_definitions(-fPIC) +add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETDB_H) +if (MSVC) + add_definitions(-DNOMINMAX) +else() + add_definitions(-DHAVE_NETINET_IN_H -fPIC) +endif() ############################################################# # Compiler flags and release types @@ -535,8 +543,10 @@ set(THRIFT_SRCS src/parquet/parquet_constants.cpp src/parquet/parquet_types.cpp) -set_source_files_properties(src/parquet/parquet_types.cpp PROPERTIES - COMPILE_FLAGS -Wno-unused-variable) +if (NOT MSVC) + set_source_files_properties(src/parquet/parquet_types.cpp PROPERTIES + COMPILE_FLAGS -Wno-unused-variable) +endif() # List of thrift output targets set(THRIFT_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/parquet) @@ -598,15 +608,11 @@ set(BUNDLED_STATIC_LIBS zlibstatic ) -add_library(parquet_objlib OBJECT - ${LIBPARQUET_SRCS} -) - # # Ensure that thrift compilation is done before using its generated headers # # in parquet code. add_custom_target(thrift-deps ALL DEPENDS ${THRIFT_OUTPUT_FILES}) -add_dependencies(parquet_objlib thrift-deps) +set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} thrift-deps) if (NOT PARQUET_MINIMAL_DEPENDENCY) # These are libraries that we will link privately with parquet_shared (as they @@ -620,57 +626,28 @@ if (NOT PARQUET_MINIMAL_DEPENDENCY) ) # Although we don't link parquet_objlib against anything, we need it to depend # on these libs as we may generate their headers via ExternalProject_Add - add_dependencies(parquet_objlib - ${LIBPARQUET_INTERFACE_LINK_LIBS}) + set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} ${LIBPARQUET_INTERFACE_LINK_LIBS}) endif() -set_property(TARGET parquet_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) - -if(APPLE) - set(SHARED_LINK_FLAGS "-undefined dynamic_lookup") -elseif() +if(NOT APPLE AND NOT MSVC) # Localize thirdparty symbols using a linker version script. This hides them # from the client application. The OS X linker does not support the # version-script option. set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/parquet/symbols.map") endif() -if (PARQUET_BUILD_SHARED) - add_library(parquet_shared SHARED $) - set_target_properties(parquet_shared - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" - LINK_FLAGS "${SHARED_LINK_FLAGS}" - OUTPUT_NAME "parquet" - VERSION "${PARQUET_ABI_VERSION}" - SOVERSION "${PARQUET_SO_VERSION}") - target_link_libraries(parquet_shared - LINK_PRIVATE ${LIBPARQUET_INTERFACE_LINK_LIBS}) - if (PARQUET_RPATH_ORIGIN) - if (APPLE) - set(_lib_install_rpath "@loader_path") - else() - set(_lib_install_rpath "\$ORIGIN") - endif() - set_target_properties(parquet_shared PROPERTIES - INSTALL_RPATH ${_lib_install_rpath}) - elseif (APPLE) - set_target_properties(parquet_shared - PROPERTIES - BUILD_WITH_INSTALL_RPATH ON - INSTALL_NAME_DIR "@rpath") - endif() -endif() - -if (PARQUET_BUILD_STATIC) - add_library(parquet_static STATIC $) - set_target_properties(parquet_static - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" - OUTPUT_NAME "parquet") - target_link_libraries(parquet_static - LINK_PUBLIC ${LIBPARQUET_INTERFACE_LINK_LIBS}) -endif() +ADD_LIB(parquet + SOURCES ${LIBPARQUET_SRCS} + LIB_BUILD_SHARED ${PARQUET_BUILD_SHARED} + LIB_BUILD_STATIC ${PARQUET_BUILD_STATIC} + DEPENDENCIES ${PARQUET_DEPENDENCIES} + SHARED_LINK_FLAGS ${SHARED_LINK_FLAGS} + SHARED_PRIVATE_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS} + STATIC_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS} + ABI_VERSION ${PARQUET_ABI_VERSION} + SO_VERSION ${PARQUET_SO_VERSION} + LIB_RPATH_ORIGIN ${PARQUET_RPATH_ORIGIN} +) add_subdirectory(src/parquet) add_subdirectory(src/parquet/api) @@ -678,7 +655,9 @@ add_subdirectory(src/parquet/column) add_subdirectory(src/parquet/file) add_subdirectory(src/parquet/util) -add_subdirectory(benchmarks) +if (NOT MSVC) + add_subdirectory(benchmarks) +endif() add_subdirectory(examples) add_subdirectory(tools) @@ -691,16 +670,3 @@ add_custom_target(clean-all COMMAND ${CMAKE_BUILD_TOOL} clean COMMAND ${CMAKE_COMMAND} -P cmake_modules/clean-all.cmake ) - -# installation - -if (PARQUET_BUILD_STATIC) - install(TARGETS parquet_static - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -endif() -if (PARQUET_BUILD_SHARED) - install(TARGETS parquet_shared - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -endif() diff --git a/LICENSE.txt b/LICENSE.txt index d1d1b705..1d879a99 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -263,3 +263,14 @@ This product includes code from the Google styleguide. Copyright: 2009 Google Inc. All rights reserved. Homepage: https://github.com/google/styleguide License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This product includes code from the Google snappy. + +* cmake_modules/SnappyCMakeLists.txt is based on the code from the Google snappy. +* cmake_modules/SnappyConfig.h is based on the code from the Google snappy. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD diff --git a/README.md b/README.md index 3b723f68..5c1f070c 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,10 @@ which is required for Thrift: brew install boost ``` +### Windows + +Check [Windows developer guide][1] for instructions to build parquet-cpp on Windows. + ## Third Party Dependencies - Apache Arrow (memory management, built-in IO, optional Array adapters) @@ -265,3 +269,5 @@ coveralls -t $PARQUET_CPP_COVERAGE_TOKEN --gcov-options '\-l' -r $PARQUET_ROOT - Note that `gcov` throws off artifacts from the STL, so I excluded my toolchain root stored in `$NATIVE_TOOLCHAIN` to avoid a cluttered coverage report. + +[1]: https://github.com/apache/parquet-cpp/blob/master/docs/Windows.md diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..437cd6d6 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Operating system (build VM template) +os: Visual Studio 2015 + +environment: + matrix: + - GENERATOR: NMake Makefiles + PYTHON: "3.5" + ARCH: "64" + MSVC_DEFAULT_OPTIONS: ON + BOOST_ROOT: C:\Libraries\boost_1_63_0 + BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0 + +init: + - set MINICONDA=C:\Miniconda35-x64 + - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% + - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 + +build_script: + - call ci\msvc-build.bat \ No newline at end of file diff --git a/benchmarks/decode_benchmark.cc b/benchmarks/decode_benchmark.cc index 5601af9e..dc687b6d 100644 --- a/benchmarks/decode_benchmark.cc +++ b/benchmarks/decode_benchmark.cc @@ -195,9 +195,9 @@ uint64_t TestPlainIntEncoding(const uint8_t* data, int num_values, int batch_siz uint64_t result = 0; parquet::PlainDecoder decoder(nullptr); decoder.SetData(num_values, data, num_values * sizeof(int64_t)); - int64_t values[batch_size]; + std::vector values(batch_size); for (int i = 0; i < num_values;) { - int n = decoder.Decode(values, batch_size); + int n = decoder.Decode(values.data(), batch_size); for (int j = 0; j < n; ++j) { result += values[j]; } @@ -247,13 +247,13 @@ uint64_t TestBinaryPackedEncoding(const char* name, const std::vector& printf(" Encoded len: %d (%0.2f%%)\n", len, len * 100 / static_cast(raw_len)); uint64_t result = 0; - int64_t buf[benchmark_batch_size]; + std::vector buf(benchmark_batch_size); parquet::StopWatch sw; sw.Start(); for (int k = 0; k < benchmark_iters; ++k) { decoder.SetData(encoder.num_values(), buffer, len); for (size_t i = 0; i < values.size();) { - int n = decoder.Decode(buf, benchmark_batch_size); + int n = decoder.Decode(buf.data(), benchmark_batch_size); for (int j = 0; j < n; ++j) { result += buf[j]; } diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat new file mode 100644 index 00000000..7e723fac --- /dev/null +++ b/ci/msvc-build.bat @@ -0,0 +1,27 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +mkdir build +cd build + +cmake -G "%GENERATOR%" ^ + -DCMAKE_BUILD_TYPE=Release ^ + .. || exit /B + +nmake || exit /B \ No newline at end of file diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake new file mode 100644 index 00000000..a6341e5f --- /dev/null +++ b/cmake_modules/BuildUtils.cmake @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(ADD_LIB LIB_NAME) + set(options) + set(one_value_args SHARED_LINK_FLAGS ABI_VERSION SO_VERSION LIB_BUILD_SHARED LIB_BUILD_STATIC LIB_RPATH_ORIGIN) + set(multi_value_args SOURCES STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS DEPENDENCIES) + cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + add_library(${LIB_NAME}_objlib OBJECT + ${ARG_SOURCES}) + + if (ARG_DEPENDENCIES) + add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES}) + endif() + + # Necessary to make static linking into other shared libraries work properly + set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) + + set(RUNTIME_INSTALL_DIR bin) + + if (ARG_LIB_BUILD_SHARED) + add_library(${LIB_NAME}_shared SHARED $) + + if(APPLE) + # On OS X, you can avoid linking at library load time and instead + # expecting that the symbols have been loaded separately. This happens + # with libpython* where there can be conflicts between system Python and + # the Python from a thirdparty distribution + set(ARG_SHARED_LINK_FLAGS + "-undefined dynamic_lookup ${ARG_SHARED_LINK_FLAGS}") + endif() + + set_target_properties(${LIB_NAME}_shared + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + PDB_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}" + OUTPUT_NAME ${LIB_NAME} + VERSION "${ARG_ABI_VERSION}" + SOVERSION "${ARG_SO_VERSION}") + + if (ARG_SHARED_LINK_LIBS OR ARG_SHARED_PRIVATE_LINK_LIBS) + target_link_libraries(${LIB_NAME}_shared + LINK_PUBLIC ${ARG_SHARED_LINK_LIBS} + LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS}) + endif() + + if (LIB_RPATH_ORIGIN) + if (APPLE) + set(_lib_install_rpath "@loader_path") + else() + set(_lib_install_rpath "\$ORIGIN") + endif() + set_target_properties(${LIB_NAME}_shared PROPERTIES + INSTALL_RPATH ${_lib_install_rpath}) + elseif (APPLE) + set_target_properties(${LIB_NAME}_shared + PROPERTIES + BUILD_WITH_INSTALL_RPATH ON + INSTALL_NAME_DIR "@rpath") + endif() + + install(TARGETS ${LIB_NAME}_shared + RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + + if (ARG_LIB_BUILD_STATIC) + if (MSVC) + set(LIB_NAME_STATIC ${LIB_NAME}_static) + else() + set(LIB_NAME_STATIC ${LIB_NAME}) + endif() + add_library(${LIB_NAME}_static STATIC $) + set_target_properties(${LIB_NAME}_static + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + OUTPUT_NAME ${LIB_NAME_STATIC}) + + if (ARG_STATIC_LINK_LIBS OR ARG_STATIC_PRIVATE_LINK_LIBS) + target_link_libraries(${LIB_NAME}_static + LINK_PUBLIC ${ARG_STATIC_LINK_LIBS} + LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS}) + endif() + + install(TARGETS ${LIB_NAME}_static + RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + +endfunction() diff --git a/cmake_modules/CompilerInfo.cmake b/cmake_modules/CompilerInfo.cmake index ecc7e385..bf59ab1f 100644 --- a/cmake_modules/CompilerInfo.cmake +++ b/cmake_modules/CompilerInfo.cmake @@ -22,8 +22,11 @@ execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v message(INFO " ${COMPILER_VERSION_FULL}") string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER) +if(MSVC) + set(COMPILER_FAMILY "msvc") + # clang on Linux and Mac OS X before 10.9 -if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*") +elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*") set(COMPILER_FAMILY "clang") string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" COMPILER_VERSION "${COMPILER_VERSION_FULL}") diff --git a/cmake_modules/SnappyCMakeLists.txt b/cmake_modules/SnappyCMakeLists.txt new file mode 100644 index 00000000..202d0634 --- /dev/null +++ b/cmake_modules/SnappyCMakeLists.txt @@ -0,0 +1,85 @@ +# Copyright 2008 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +PROJECT(snappy) + +INCLUDE(CheckIncludeFiles) +INCLUDE(CMakePackageConfigHelpers) + +CHECK_INCLUDE_FILES("stdint.h" HAVE_STDINT_H) +CHECK_INCLUDE_FILES("stddef.h" HAVE_STDDEF_H) +CHECK_INCLUDE_FILES("sys/uio.h" HAVE_SYS_UIO_H) + +if (NOT HAVE_SYS_UIO_H) + set(HAVE_SYS_UIO_H 0) +endif() + +if (NOT HAVE_STDINT_H) + set(HAVE_STDINT_H 0) +endif() + +if (NOT HAVE_STDDEF_H) + set(HAVE_STDDEF_H 0) +endif() + +set(ac_cv_have_stdint_h ${HAVE_STDINT_H}) +set(ac_cv_have_stddef_h ${HAVE_STDDEF_H}) +set(ac_cv_have_sys_uio_h ${HAVE_SYS_UIO_H}) +CONFIGURE_FILE(${snappy_SOURCE_DIR}/snappy-stubs-public.h.in + snappy-stubs-public.h) + +if (WIN32) + ADD_DEFINITIONS(-D_CRT_SECURE_NO_WARNINGS) +endif() + +set(SNAPPY_SRCS snappy.cc + snappy-c.cc + snappy-stubs-internal.cc + snappy-sinksource.cc + snappy.h + snappy-c.h + snappy-sinksource.h + snappy-stubs-public.h) + +add_library(snappy SHARED ${SNAPPY_SRCS}) +add_library(snappystatic STATIC ${SNAPPY_SRCS}) + +TARGET_COMPILE_DEFINITIONS(snappy PRIVATE -DHAVE_CONFIG_H) +TARGET_COMPILE_DEFINITIONS(snappystatic PRIVATE -DHAVE_CONFIG_H) + +install(FILES snappy.h + snappy-c.h + snappy-sinksource.h + ${snappy_BINARY_DIR}/snappy-stubs-public.h + DESTINATION include) + +install(TARGETS snappy snappystatic + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) \ No newline at end of file diff --git a/cmake_modules/SnappyConfig.h b/cmake_modules/SnappyConfig.h new file mode 100644 index 00000000..74eb7762 --- /dev/null +++ b/cmake_modules/SnappyConfig.h @@ -0,0 +1,36 @@ +// Copyright 2008 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef SNAPPY_CONFIG_H +#define SNAPPY_CONFIG_H 1 + +#if defined(_MSC_VER) && (_MSC_VER <= 1900) +typedef __int64 ssize_t; +#endif + +#endif diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 478e018a..930a42f5 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -72,6 +72,15 @@ set(Boost_USE_MULTITHREADED ON) if (PARQUET_BOOST_USE_SHARED) # Find shared Boost libraries. set(Boost_USE_STATIC_LIBS OFF) + + if (MSVC) + # disable autolinking in boost + add_definitions(-DBOOST_ALL_NO_LIB) + + # force all boost libraries to dynamic link + add_definitions(-DBOOST_ALL_DYN_LINK) + endif() + find_package(Boost COMPONENTS regex REQUIRED) if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) @@ -93,7 +102,13 @@ message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS}) message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) if (PARQUET_BOOST_USE_SHARED) add_library(boost_shared_regex SHARED IMPORTED) - set_target_properties(boost_shared_regex PROPERTIES IMPORTED_LOCATION ${BOOST_SHARED_REGEX_LIBRARY}) + if (MSVC) + set_target_properties(boost_shared_regex + PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_REGEX_LIBRARY}") + else() + set_target_properties(boost_shared_regex + PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_REGEX_LIBRARY}") + endif() else() add_library(boost_static_regex STATIC IMPORTED) set_target_properties(boost_static_regex PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_REGEX_LIBRARY}) @@ -102,6 +117,52 @@ endif() include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) set(LIBS ${LIBS} ${Boost_LIBRARIES}) +# ---------------------------------------------------------------------- +# ZLIB + +if (NOT PARQUET_ZLIB_VENDORED) + find_package(ZLIB) +endif() + +if (NOT ZLIB_FOUND) + set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") + set(ZLIB_HOME "${ZLIB_PREFIX}") + set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include") + if (MSVC) + set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) + else() + set(ZLIB_STATIC_LIB_NAME libz.a) + endif() + set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") + set(ZLIB_VENDORED 1) + set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DBUILD_SHARED_LIBS=OFF) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(zlib_ep + URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz" + BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) + else() + ExternalProject_Add(zlib_ep + URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) + endif() +else() + set(ZLIB_VENDORED 0) +endif() + +include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) +add_library(zlibstatic STATIC IMPORTED) +set_target_properties(zlibstatic PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB}) + +if (ZLIB_VENDORED) + add_dependencies(zlibstatic zlib_ep) +endif() + # ---------------------------------------------------------------------- # Thrift @@ -116,7 +177,11 @@ if (NOT THRIFT_FOUND) IF (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a") ELSE() - set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthrift.a") + IF (MSVC) + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmd.lib") + ELSE() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthrift.a") + ENDIF() ENDIF() set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") set(THRIFT_VENDORED 1) @@ -134,23 +199,66 @@ if (NOT THRIFT_FOUND) "-DWITH_CPP=ON" "-DWITH_STATIC_LIB=ON" ) + if (MSVC) + set (THRIFT_CMAKE_ARGS "-DLIBEVENT_ROOT=${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep/thirdparty/src/Libevent-release-2.1.7-rc" + "-DFLEX_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep/thirdparty/dist/winflexbison/win_flex.exe" + "-DBISON_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep/thirdparty/dist/winflexbison/win_bison.exe" + "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS}" + "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" + "-DWITH_SHARED_LIB=OFF" + ${THRIFT_CMAKE_ARGS}) + endif() if (CMAKE_VERSION VERSION_GREATER "3.2") # BUILD_BYPRODUCTS is a 3.2+ feature ExternalProject_Add(thrift_ep URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}" - CMAKE_ARGS ${THRIFT_CMAKE_ARGS}) + CMAKE_ARGS ${THRIFT_CMAKE_ARGS} + STEP_TARGETS flex_step libevent_step) else() ExternalProject_Add(thrift_ep URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" - CMAKE_ARGS ${THRIFT_CMAKE_ARGS}) + CMAKE_ARGS ${THRIFT_CMAKE_ARGS} + STEP_TARGETS flex_step libevent_step) endif() set(THRIFT_VENDORED 1) else() set(THRIFT_VENDORED 0) endif() +if (MSVC) + ExternalProject_Get_Property(thrift_ep SOURCE_DIR) + + set(WINFLEXBISON_VERSION 2.4.9) + set(LIBEVENT_VERSION 2.1.7) + + # Download and configure Windows build of Flex and Bison + ExternalProject_Add_Step(thrift_ep flex_step + COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison && + cd thirdparty/dist/winflexbison && + curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip && + ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip + DEPENDERS configure + WORKING_DIRECTORY ${SOURCE_DIR}) + + # Download and build libevent + ExternalProject_Add_Step(thrift_ep libevent_step + COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src && + cd thirdparty/src && + curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip && + ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip && + cd Libevent-release-${LIBEVENT_VERSION}-rc && + nmake -f Makefile.nmake && + ${CMAKE_COMMAND} -E make_directory lib && + copy *.lib lib && + xcopy /E /I /D WIN32-Code\\nmake include\\event2 && + xcopy /E /I /D WIN32-Code\\nmake\\event2 include\\event2 && + copy *.h include + DEPENDERS configure + WORKING_DIRECTORY ${SOURCE_DIR}) +endif() + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR} ${THRIFT_INCLUDE_DIR}/thrift) message(STATUS "Thrift include dir: ${THRIFT_INCLUDE_DIR}") message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}") @@ -171,8 +279,14 @@ if (NOT SNAPPY_FOUND) set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install") set(SNAPPY_HOME "${SNAPPY_PREFIX}") set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include") - set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/libsnappy.a") + if (MSVC) + set(SNAPPY_STATIC_LIB_NAME snappystatic) + else() + set(SNAPPY_STATIC_LIB_NAME snappy) + endif() + set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") set(SNAPPY_VENDORED 1) + set(SNAPPY_SRC_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz") if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE") if (APPLE) @@ -182,24 +296,54 @@ if (NOT SNAPPY_FOUND) endif() endif() - if (CMAKE_VERSION VERSION_GREATER "3.2") - # BUILD_BYPRODUCTS is a 3.2+ feature - ExternalProject_Add(snappy_ep - CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} - BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} - INSTALL_DIR ${SNAPPY_PREFIX} - URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" - BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}" - ) + if (MSVC) + set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS=${EX_C_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") + set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt + ./CMakeLists.txt && + ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h + ./config.h) + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(snappy_ep + UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND} + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SRC_URL} + CMAKE_ARGS ${SNAPPY_CMAKE_ARGS} + BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") + else() + ExternalProject_Add(snappy_ep + UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND} + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SRC_URL} + CMAKE_ARGS ${SNAPPY_CMAKE_ARGS}) + endif() else() - ExternalProject_Add(snappy_ep - CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} - BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} - INSTALL_DIR ${SNAPPY_PREFIX} - URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" - ) + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(snappy_ep + CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SRC_URL} + BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") + else() + ExternalProject_Add(snappy_ep + CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SRC_URL}) + endif() endif() else() set(SNAPPY_VENDORED 0) @@ -221,9 +365,14 @@ if (NOT BROTLI_FOUND) set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") set(BROTLI_HOME "${BROTLI_PREFIX}") set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") - set(BROTLI_LIBRARY_ENC "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlienc.a") - set(BROTLI_LIBRARY_DEC "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlidec.a") - set(BROTLI_LIBRARY_COMMON "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlicommon.a") + if (MSVC) + set(BROTLI_LIB_DIR bin) + else() + set(BROTLI_LIB_DIR lib) + endif() + set(BROTLI_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") set(BROTLI_VENDORED 1) set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" @@ -237,7 +386,8 @@ if (NOT BROTLI_FOUND) ExternalProject_Add(brotli_ep URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz" BUILD_BYPRODUCTS "${BROTLI_LIBRARY_ENC}" "${BROTLI_LIBRARY_DEC}" "${BROTLI_LIBRARY_COMMON}" - CMAKE_ARGS ${BROTLI_CMAKE_ARGS}) + CMAKE_ARGS ${BROTLI_CMAKE_ARGS} + STEP_TARGETS headers_copy) else() ExternalProject_Add(brotli_ep URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz" @@ -247,6 +397,15 @@ else() set(BROTLI_VENDORED 0) endif() +if (MSVC) + ExternalProject_Get_Property(brotli_ep SOURCE_DIR) + + ExternalProject_Add_Step(brotli_ep headers_copy + COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y + DEPENDEES build + WORKING_DIRECTORY ${SOURCE_DIR}) +endif() + include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) add_library(brotlistatic_enc STATIC IMPORTED) set_target_properties(brotlistatic_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_LIBRARY_ENC}) @@ -261,47 +420,6 @@ if (BROTLI_VENDORED) add_dependencies(brotlistatic_common brotli_ep) endif() -# ---------------------------------------------------------------------- -# ZLIB - -if (NOT PARQUET_ZLIB_VENDORED) - find_package(ZLIB) -endif() - -if (NOT ZLIB_FOUND) - set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") - set(ZLIB_HOME "${ZLIB_PREFIX}") - set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include") - set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/libz.a") - set(ZLIB_VENDORED 1) - set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DBUILD_SHARED_LIBS=OFF) - - if (CMAKE_VERSION VERSION_GREATER "3.2") - # BUILD_BYPRODUCTS is a 3.2+ feature - ExternalProject_Add(zlib_ep - URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz" - BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" - CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - else() - ExternalProject_Add(zlib_ep - URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz" - CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - endif() -else() - set(ZLIB_VENDORED 0) -endif() - -include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) -add_library(zlibstatic STATIC IMPORTED) -set_target_properties(zlibstatic PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB}) - -if (ZLIB_VENDORED) - add_dependencies(zlibstatic zlib_ep) -endif() - ## GTest if(PARQUET_BUILD_TESTS AND NOT IGNORE_OPTIONAL_PACKAGES) add_custom_target(unittest ctest -L unittest) @@ -417,8 +535,16 @@ if (NOT ARROW_FOUND) set(ARROW_HOME "${ARROW_PREFIX}") set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib") - set(ARROW_SHARED_LIB "${ARROW_LIB_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(ARROW_STATIC_LIB "${ARROW_LIB_DIR}/libarrow.a") + set(ARROW_BIN_DIR "${ARROW_PREFIX}/bin") + if (MSVC) + set(ARROW_SHARED_LIB "${ARROW_BIN_DIR}/arrow.dll") + set(ARROW_SHARED_IMPLIB "${ARROW_LIB_DIR}/arrow.lib") + set(ARROW_STATIC_LIB "${ARROW_LIB_DIR}/arrow_static.lib") + else() + set(ARROW_SHARED_LIB "${ARROW_LIB_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(ARROW_STATIC_LIB "${ARROW_LIB_DIR}/libarrow.a") + endif() + set(ARROW_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} @@ -427,6 +553,10 @@ if (NOT ARROW_FOUND) -DARROW_JEMALLOC=OFF -DARROW_IPC=OFF -DARROW_BUILD_TESTS=OFF) + + if (MSVC) + set(ARROW_CMAKE_ARGS -G "NMake Makefiles" ${ARROW_CMAKE_ARGS}) + endif() if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "") set(ARROW_VERSION "f7ab7270bb07466dabf84c015a6db2a192eb3dad") @@ -437,22 +567,54 @@ if (NOT ARROW_FOUND) set(ARROW_URL "https://github.com/apache/arrow/archive/${ARROW_VERSION}.tar.gz") - if (CMAKE_VERSION VERSION_GREATER "3.2") - # BUILD_BYPRODUCTS is a 3.2+ feature - ExternalProject_Add(arrow_ep - URL ${ARROW_URL} - BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_STATIC_LIB}" - # With CMake 3.7.0 there is a SOURCE_SUBDIR argument which we can use - # to specify that the CMakeLists.txt of Arrow is located in cpp/ - # - # See https://gitlab.kitware.com/cmake/cmake/commit/a8345d65f359d75efb057d22976cfb92b4d477cf - CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp - CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + if (MSVC) + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(arrow_ep + URL ${ARROW_URL} + BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_STATIC_LIB}" + # With CMake 3.7.0 there is a SOURCE_SUBDIR argument which we can use + # to specify that the CMakeLists.txt of Arrow is located in cpp/ + # + # See https://gitlab.kitware.com/cmake/cmake/commit/a8345d65f359d75efb057d22976cfb92b4d477cf + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_GENERATOR "NMake Makefiles" + CMAKE_GENERATOR_PLATFORM "x64" + BUILD_COMMAND nmake && nmake install + STEP_TARGETS copy_dll_step) + else() + ExternalProject_Add(arrow_ep + URL ${ARROW_URL} + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_GENERATOR "NMake Makefiles" + CMAKE_GENERATOR_PLATFORM "x64" + BUILD_COMMAND nmake && nmake install + STEP_TARGETS copy_dll_step) + endif() + + ExternalProject_Get_Property(arrow_ep SOURCE_DIR) + ExternalProject_Add_Step(arrow_ep copy_dll_step + COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY} + DEPENDEES build + WORKING_DIRECTORY ${SOURCE_DIR}) else() - ExternalProject_Add(arrow_ep - URL ${ARROW_URL} - CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp - CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(arrow_ep + URL ${ARROW_URL} + BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_STATIC_LIB}" + # With CMake 3.7.0 there is a SOURCE_SUBDIR argument which we can use + # to specify that the CMakeLists.txt of Arrow is located in cpp/ + # + # See https://gitlab.kitware.com/cmake/cmake/commit/a8345d65f359d75efb057d22976cfb92b4d477cf + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + else() + ExternalProject_Add(arrow_ep + URL ${ARROW_URL} + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + endif() endif() set(ARROW_VENDORED 1) else() @@ -461,7 +623,13 @@ endif() include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) add_library(arrow SHARED IMPORTED) -set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_SHARED_LIB}) +if(MSVC) + set_target_properties(arrow + PROPERTIES IMPORTED_IMPLIB "${ARROW_SHARED_IMPLIB}") +else() + set_target_properties(arrow + PROPERTIES IMPORTED_LOCATION "${ARROW_SHARED_LIB}") +endif() add_library(arrow_static STATIC IMPORTED) set_target_properties(arrow_static PROPERTIES IMPORTED_LOCATION ${ARROW_STATIC_LIB}) diff --git a/docs/Windows.md b/docs/Windows.md new file mode 100644 index 00000000..8c7d1a3a --- /dev/null +++ b/docs/Windows.md @@ -0,0 +1,76 @@ + + +# Building parquet-cpp on Windows + +## Fast setup of building requirements with conda and conda-forge + +A convenient and tested way to set up thirdparty dependencies is to use the +conda package manager. +Please feel free to extend this document with others ways to setup +development environment for parquet-cpp. + +### conda and package toolchain + +[Miniconda][1] is a minimal Python distribution including the conda package +manager. To get started, download and install a 64-bit distribution. + +We recommend using packages from [conda-forge][2]. +Launch cmd.exe and run following to bootstrap a build environment: + +```shell +conda create -n parquet-dev cmake git boost-cpp curl zlib snappy -c conda-forge +``` + +### Visual Studio + +Microsoft provides the free Visual Studio Community edition. Once you have +Visual Studio installed, you should configure cmd.exe environment to be able +to find Visual Studio's build toolchain by running following commands: + +#### Visual Studio 2015 + +```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64``` + +#### Visual Studio 2017 + +```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64``` + +It's easiest to configure a console emulator like [cmder][3] to automatically +launch this when starting a new development console. + +## Building with NMake + +Activate your conda build environment: + +``` +activate parquet-dev +``` + +Change working directory in cmd.exe to the root directory of parquet-cpp and +do an out of source build using `nmake`: + +``` +cd %PARQUET_ROOT_SOURCES_DIRECTORY% +mkdir build +cd build +cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release .. +nmake +``` + +When using conda, only release builds are currently supported. + +[1]: https://conda.io/miniconda.html +[2]: https://conda-forge.github.io/ +[3]: http://cmder.net/ \ No newline at end of file diff --git a/src/parquet/arrow/CMakeLists.txt b/src/parquet/arrow/CMakeLists.txt index c2fd9013..8f2a698f 100644 --- a/src/parquet/arrow/CMakeLists.txt +++ b/src/parquet/arrow/CMakeLists.txt @@ -24,62 +24,23 @@ set(PARQUET_ARROW_SRCS writer.cc ) -add_library(parquet_arrow_objlib OBJECT - ${PARQUET_ARROW_SRCS} -) - -# Add dependencies so ExternalProjects are built beforehand -add_dependencies(parquet_arrow_objlib - arrow_static - parquet_static) - -# SET_TARGET_PROPERTIES(parquet_arrow PROPERTIES LINKER_LANGUAGE CXX) +# Set dependencies so ExternalProjects are built beforehand +set(PARQUET_ARROW_DEPENDENCIES ${ARROW_LINK_LIBS} parquet_static) +SET(PARQUET_ARROW_SHARED_LINK_LIBS ${ARROW_LINK_LIBS} parquet_shared) +SET(PARQUET_ARROW_STATIC_LINK_LIBS ${ARROW_LINK_LIBS} parquet_static) -if (PARQUET_BUILD_SHARED) - add_library(parquet_arrow_shared SHARED $) - set_target_properties(parquet_arrow_shared - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" - LINK_FLAGS "${SHARED_LINK_FLAGS}" - OUTPUT_NAME "parquet_arrow" - VERSION "${PARQUET_ABI_VERSION}" - SOVERSION "${PARQUET_SO_VERSION}") - target_link_libraries(parquet_arrow_shared - arrow - parquet_shared) - if (PARQUET_RPATH_ORIGIN) - if (APPLE) - set(_lib_install_rpath "@loader_path") - else() - set(_lib_install_rpath "\$ORIGIN") - endif() - set_target_properties(parquet_arrow_shared PROPERTIES - INSTALL_RPATH ${_lib_install_rpath}) - elseif (APPLE) - set_target_properties(parquet_arrow_shared - PROPERTIES - BUILD_WITH_INSTALL_RPATH ON - INSTALL_NAME_DIR "@rpath") - endif() - - install(TARGETS parquet_arrow_shared - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -endif() - -if (PARQUET_BUILD_STATIC) - add_library(parquet_arrow_static STATIC $) - set_target_properties(parquet_arrow_static - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" - OUTPUT_NAME "parquet_arrow") - target_link_libraries(parquet_arrow_static - arrow_static - parquet_static) - install(TARGETS parquet_arrow_static - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -endif() +ADD_LIB(parquet_arrow + SOURCES ${PARQUET_ARROW_SRCS} + LIB_BUILD_SHARED ${PARQUET_BUILD_SHARED} + LIB_BUILD_STATIC ${PARQUET_BUILD_STATIC} + DEPENDENCIES ${PARQUET_ARROW_DEPENDENCIES} + SHARED_LINK_FLAGS ${SHARED_LINK_FLAGS} + SHARED_LINK_LIBS ${PARQUET_ARROW_SHARED_LINK_LIBS} + STATIC_LINK_LIBS ${PARQUET_ARROW_STATIC_LINK_LIBS} + ABI_VERSION ${PARQUET_ABI_VERSION} + SO_VERSION ${PARQUET_SO_VERSION} + LIB_RPATH_ORIGIN ${PARQUET_RPATH_ORIGIN} +) ADD_PARQUET_TEST(arrow-schema-test) ADD_PARQUET_TEST(arrow-reader-writer-test) diff --git a/src/parquet/column/levels.h b/src/parquet/column/levels.h index f676db23..63e325e6 100644 --- a/src/parquet/column/levels.h +++ b/src/parquet/column/levels.h @@ -31,7 +31,7 @@ class BitWriter; class RleDecoder; class RleEncoder; -class LevelEncoder { +class PARQUET_EXPORT LevelEncoder { public: LevelEncoder(); ~LevelEncoder(); @@ -61,7 +61,7 @@ class LevelEncoder { std::unique_ptr bit_packed_encoder_; }; -class LevelDecoder { +class PARQUET_EXPORT LevelDecoder { public: LevelDecoder(); ~LevelDecoder(); diff --git a/src/parquet/compression.h b/src/parquet/compression.h index abd48996..f006e54c 100644 --- a/src/parquet/compression.h +++ b/src/parquet/compression.h @@ -46,7 +46,7 @@ class Codec { }; // Snappy codec. -class SnappyCodec : public Codec { +class PARQUET_EXPORT SnappyCodec : public Codec { public: virtual void Decompress(int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer); @@ -60,7 +60,7 @@ class SnappyCodec : public Codec { }; // Brotli codec. -class BrotliCodec : public Codec { +class PARQUET_EXPORT BrotliCodec : public Codec { public: void Decompress(int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) override; @@ -74,7 +74,7 @@ class BrotliCodec : public Codec { }; // GZip codec. -class GZipCodec : public Codec { +class PARQUET_EXPORT GZipCodec : public Codec { public: /// Compression formats supported by the zlib library enum Format { diff --git a/src/parquet/encoding-internal.h b/src/parquet/encoding-internal.h index dc2c336d..7e902548 100644 --- a/src/parquet/encoding-internal.h +++ b/src/parquet/encoding-internal.h @@ -852,8 +852,8 @@ class DeltaLengthByteArrayDecoder : public Decoder { virtual int Decode(ByteArray* buffer, int max_values) { max_values = std::min(max_values, num_values_); - int lengths[max_values]; - len_decoder_.Decode(lengths, max_values); + std::vector lengths(max_values); + len_decoder_.Decode(lengths.data(), max_values); for (int i = 0; i < max_values; ++i) { buffer[i].len = lengths[i]; buffer[i].ptr = data_; diff --git a/src/parquet/file/reader-internal.h b/src/parquet/file/reader-internal.h index 0d0fb073..5d35540d 100644 --- a/src/parquet/file/reader-internal.h +++ b/src/parquet/file/reader-internal.h @@ -30,6 +30,7 @@ #include "parquet/parquet_types.h" #include "parquet/types.h" #include "parquet/util/memory.h" +#include "parquet/util/visibility.h" namespace parquet { @@ -42,7 +43,7 @@ static constexpr uint32_t DEFAULT_PAGE_HEADER_SIZE = 16 * 1024; // This subclass delimits pages appearing in a serialized stream, each preceded // by a serialized Thrift format::PageHeader indicating the type of each page // and the page metadata. -class SerializedPageReader : public PageReader { +class PARQUET_EXPORT SerializedPageReader : public PageReader { public: SerializedPageReader(std::unique_ptr stream, int64_t num_rows, Compression::type codec, @@ -76,7 +77,7 @@ class SerializedPageReader : public PageReader { }; // RowGroupReader::Contents implementation for the Parquet file specification -class SerializedRowGroup : public RowGroupReader::Contents { +class PARQUET_EXPORT SerializedRowGroup : public RowGroupReader::Contents { public: SerializedRowGroup(RandomAccessSource* source, FileMetaData* file_metadata, int row_group_number, const ReaderProperties& props); @@ -97,7 +98,7 @@ class SerializedRowGroup : public RowGroupReader::Contents { // An implementation of ParquetFileReader::Contents that deals with the Parquet // file structure, Thrift deserialization, and other internal matters -class SerializedFile : public ParquetFileReader::Contents { +class PARQUET_EXPORT SerializedFile : public ParquetFileReader::Contents { public: // Open the file. If no metadata is passed, it is parsed from the footer of // the file diff --git a/src/parquet/parquet.thrift b/src/parquet/parquet.thrift index b61c084e..2e840d89 100644 --- a/src/parquet/parquet.thrift +++ b/src/parquet/parquet.thrift @@ -17,6 +17,8 @@ * under the License. */ +cpp_include "parquet/util/windows_compatibility.h" + /** * File format description for the parquet file format */ diff --git a/src/parquet/util/bit-util.h b/src/parquet/util/bit-util.h index 3a0f5876..8f0a2709 100644 --- a/src/parquet/util/bit-util.h +++ b/src/parquet/util/bit-util.h @@ -22,10 +22,20 @@ #if defined(__APPLE__) #include +#elif defined(_WIN32) +#define __LITTLE_ENDIAN 1 #else #include #endif +#if defined(_MSC_VER) +#define PARQUET_BYTE_SWAP64 _byteswap_uint64 +#define PARQUET_BYTE_SWAP32 _byteswap_ulong +#else +#define PARQUET_BYTE_SWAP64 __builtin_bswap64 +#define PARQUET_BYTE_SWAP32 __builtin_bswap32 +#endif + #include #include "parquet/util/compiler-util.h" @@ -199,13 +209,13 @@ class BitUtil { } /// Swaps the byte order (i.e. endianess) - static inline int64_t ByteSwap(int64_t value) { return __builtin_bswap64(value); } + static inline int64_t ByteSwap(int64_t value) { return PARQUET_BYTE_SWAP64(value); } static inline uint64_t ByteSwap(uint64_t value) { - return static_cast(__builtin_bswap64(value)); + return static_cast(PARQUET_BYTE_SWAP64(value)); } - static inline int32_t ByteSwap(int32_t value) { return __builtin_bswap32(value); } + static inline int32_t ByteSwap(int32_t value) { return PARQUET_BYTE_SWAP32(value); } static inline uint32_t ByteSwap(uint32_t value) { - return static_cast(__builtin_bswap32(value)); + return static_cast(PARQUET_BYTE_SWAP32(value)); } static inline int16_t ByteSwap(int16_t value) { return (((value >> 8) & 0xff) | ((value & 0xff) << 8)); diff --git a/src/parquet/util/compiler-util.h b/src/parquet/util/compiler-util.h index 3f2c3730..9d0b2650 100644 --- a/src/parquet/util/compiler-util.h +++ b/src/parquet/util/compiler-util.h @@ -31,8 +31,13 @@ #undef UNLIKELY #endif +#ifdef _MSC_VER +#define LIKELY(expr) expr +#define UNLIKELY(expr) expr +#else #define LIKELY(expr) __builtin_expect(!!(expr), 1) #define UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#endif #define PREFETCH(addr) __builtin_prefetch(addr) diff --git a/src/parquet/util/cpu-info.cc b/src/parquet/util/cpu-info.cc index 06304c10..0e192234 100644 --- a/src/parquet/util/cpu-info.cc +++ b/src/parquet/util/cpu-info.cc @@ -26,7 +26,10 @@ #include #include + +#ifndef _MSC_VER #include +#endif #include diff --git a/src/parquet/util/cpu-info.h b/src/parquet/util/cpu-info.h index ae59ef02..4d7cd273 100644 --- a/src/parquet/util/cpu-info.h +++ b/src/parquet/util/cpu-info.h @@ -24,13 +24,15 @@ #include #include +#include "parquet/util/visibility.h" + namespace parquet { /// CpuInfo is an interface to query for cpu information at runtime. The caller can /// ask for the sizes of the caches and what hardware features are supported. /// On Linux, this information is pulled from a couple of sys files (/proc/cpuinfo and /// /sys/devices) -class CpuInfo { +class PARQUET_EXPORT CpuInfo { public: static const int64_t SSSE3 = (1 << 1); static const int64_t SSE4_1 = (1 << 2); diff --git a/src/parquet/util/memory.h b/src/parquet/util/memory.h index 4a154305..45d8bf04 100644 --- a/src/parquet/util/memory.h +++ b/src/parquet/util/memory.h @@ -68,7 +68,7 @@ using ResizableBuffer = ::arrow::ResizableBuffer; using PoolBuffer = ::arrow::PoolBuffer; template -class Vector { +class PARQUET_EXPORT Vector { public: explicit Vector(int64_t size, ::arrow::MemoryPool* pool); void Resize(int64_t new_size); @@ -127,7 +127,7 @@ class Vector { /// The one remaining (empty) chunk is released: /// delete p; -class ChunkedAllocator { +class PARQUET_EXPORT ChunkedAllocator { public: explicit ChunkedAllocator(::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); @@ -390,7 +390,7 @@ class InputStream { }; // Implementation of an InputStream when all the bytes are in memory. -class InMemoryInputStream : public InputStream { +class PARQUET_EXPORT InMemoryInputStream : public InputStream { public: InMemoryInputStream(RandomAccessSource* source, int64_t start, int64_t end); explicit InMemoryInputStream(const std::shared_ptr& buffer); @@ -406,7 +406,7 @@ class InMemoryInputStream : public InputStream { }; // Implementation of an InputStream when only some of the bytes are in memory. -class BufferedInputStream : public InputStream { +class PARQUET_EXPORT BufferedInputStream : public InputStream { public: BufferedInputStream(::arrow::MemoryPool* pool, int64_t buffer_size, RandomAccessSource* source, int64_t start, int64_t end); @@ -424,9 +424,10 @@ class BufferedInputStream : public InputStream { int64_t buffer_size_; }; -std::shared_ptr AllocateBuffer(::arrow::MemoryPool* pool, int64_t size = 0); +std::shared_ptr PARQUET_EXPORT AllocateBuffer( + ::arrow::MemoryPool* pool, int64_t size = 0); -std::unique_ptr AllocateUniqueBuffer( +std::unique_ptr PARQUET_EXPORT AllocateUniqueBuffer( ::arrow::MemoryPool* pool, int64_t size = 0); } // namespace parquet diff --git a/src/parquet/util/rle-test.cc b/src/parquet/util/rle-test.cc index e50cd72a..86c05bc5 100644 --- a/src/parquet/util/rle-test.cc +++ b/src/parquet/util/rle-test.cc @@ -103,11 +103,12 @@ TEST(BitArray, TestBool) { // Writes 'num_vals' values with width 'bit_width' and reads them back. void TestBitArrayValues(int bit_width, int num_vals) { - const int len = BitUtil::Ceil(bit_width * num_vals, 8); + int len = BitUtil::Ceil(bit_width * num_vals, 8); + EXPECT_TRUE(len > 0); const uint64_t mod = bit_width == 64 ? 1 : 1LL << bit_width; - uint8_t buffer[len]; - BitWriter writer(buffer, len); + std::vector buffer(len); + BitWriter writer(buffer.data(), len); for (int i = 0; i < num_vals; ++i) { bool result = writer.PutValue(i % mod, bit_width); EXPECT_TRUE(result); @@ -115,7 +116,7 @@ void TestBitArrayValues(int bit_width, int num_vals) { writer.Flush(); EXPECT_EQ(writer.bytes_written(), len); - BitReader reader(buffer, len); + BitReader reader(buffer.data(), len); for (int i = 0; i < num_vals; ++i) { int64_t val = 0; bool result = reader.GetValue(bit_width, &val); @@ -126,7 +127,7 @@ void TestBitArrayValues(int bit_width, int num_vals) { } TEST(BitArray, TestValues) { - for (int width = 0; width <= MAX_WIDTH; ++width) { + for (int width = 1; width <= MAX_WIDTH; ++width) { TestBitArrayValues(width, 1); TestBitArrayValues(width, 2); // Don't write too many values @@ -421,12 +422,12 @@ TEST(BitRle, RepeatedPattern) { TEST(BitRle, Overflow) { for (int bit_width = 1; bit_width < 32; bit_width += 3) { - const int len = RleEncoder::MinBufferSize(bit_width); - uint8_t buffer[len]; + int len = RleEncoder::MinBufferSize(bit_width); + std::vector buffer(len); int num_added = 0; bool parity = true; - RleEncoder encoder(buffer, len, bit_width); + RleEncoder encoder(buffer.data(), len, bit_width); // Insert alternating true/false until there is no space left while (true) { bool result = encoder.Put(parity); @@ -439,7 +440,7 @@ TEST(BitRle, Overflow) { EXPECT_LE(bytes_written, len); EXPECT_GT(num_added, 0); - RleDecoder decoder(buffer, bytes_written, bit_width); + RleDecoder decoder(buffer.data(), bytes_written, bit_width); parity = true; uint32_t v; for (int i = 0; i < num_added; ++i) { diff --git a/src/parquet/util/stopwatch.h b/src/parquet/util/stopwatch.h index ad502676..68cf792f 100644 --- a/src/parquet/util/stopwatch.h +++ b/src/parquet/util/stopwatch.h @@ -19,7 +19,9 @@ #define PARQUET_UTIL_STOPWATCH_H #include +#ifndef _MSC_VER #include +#endif #include #include diff --git a/src/parquet/util/visibility.h b/src/parquet/util/visibility.h index 64eef90c..ccaefd89 100644 --- a/src/parquet/util/visibility.h +++ b/src/parquet/util/visibility.h @@ -20,6 +20,7 @@ #if defined(_WIN32) || defined(__CYGWIN__) #define PARQUET_EXPORT __declspec(dllexport) +#define PARQUET_NO_EXPORT #else // Not Windows #ifndef PARQUET_EXPORT #define PARQUET_EXPORT __attribute__((visibility("default"))) diff --git a/src/parquet/util/windows_compatibility.h b/src/parquet/util/windows_compatibility.h new file mode 100644 index 00000000..899590ac --- /dev/null +++ b/src/parquet/util/windows_compatibility.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PARQUET_UTIL_WINDOWS_COMPATIBILITY +#define PARQUET_UTIL_WINDOWS_COMPATIBILITY + +#ifdef _WIN32 + +// Windows defines min and max macros that mess up std::min/max +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#include +#include + +#ifdef OPTIONAL +#undef OPTIONAL +#endif + +#endif // _WIN32 + +#endif // PARQUET_UTIL_WINDOWS_COMPATIBILITY diff --git a/tools/parquet-scan.cc b/tools/parquet-scan.cc index f0bbb8e1..8ab15a4d 100644 --- a/tools/parquet-scan.cc +++ b/tools/parquet-scan.cc @@ -71,7 +71,7 @@ int main(int argc, char** argv) { } } - int64_t total_rows[num_columns]; + std::vector total_rows(num_columns); for (int r = 0; r < reader->metadata()->num_row_groups(); ++r) { auto group_reader = reader->RowGroup(r);