diff --git a/.env b/.env index 62f5ae52b71..9e228b6aa53 100644 --- a/.env +++ b/.env @@ -24,9 +24,9 @@ ARCH=amd64 CUDA=9.1 DEBIAN=10 UBUNTU=18.04 -FEDORA=32 +FEDORA=33 PYTHON=3.6 -LLVM=10 +LLVM=11 CLANG_TOOLS=8 RUST=nightly-2020-04-22 GO=1.12 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index d8eab595905..b66334c6e9c 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -336,7 +336,6 @@ jobs: CMAKE_ARGS: >- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} -DBoost_NO_BOOST_CMAKE=ON - CMAKE_GENERATOR: MSYS Makefiles CMAKE_UNITY_BUILD: ON steps: - name: Disable Crash Dialogs diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml index 95c5f4b7cbb..5f404b22fe5 100644 --- a/.github/workflows/cpp_cron.yml +++ b/.github/workflows/cpp_cron.yml @@ -46,7 +46,7 @@ jobs: matrix: name: - amd64-debian-10-cpp - - amd64-fedora-32-cpp + - amd64-fedora-33-cpp - amd64-ubuntu-16.04-cpp - amd64-ubuntu-18.04-cpp - amd64-ubuntu-18.04-cpp-cmake32 @@ -55,10 +55,10 @@ jobs: image: debian-cpp title: AMD64 Debian 10 C++ debian: 10 - - name: amd64-fedora-32-cpp + - name: amd64-fedora-33-cpp image: fedora-cpp - title: AMD64 Fedora 32 C++ - fedora: 32 + title: AMD64 Fedora 33 C++ + fedora: 33 - name: amd64-ubuntu-16.04-cpp image: ubuntu-cpp title: AMD64 Ubuntu 16.04 C++ @@ -75,7 +75,7 @@ jobs: # the defaults here should correspond to the values in .env ARCH: 'amd64' DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 32 }} + FEDORA: ${{ matrix.fedora || 33 }} UBUNTU: ${{ matrix.ubuntu || 18.04 }} steps: - name: Checkout Arrow diff --git a/.github/workflows/python_cron.yml b/.github/workflows/python_cron.yml index 133a73b59bf..256856bd197 100644 --- a/.github/workflows/python_cron.yml +++ b/.github/workflows/python_cron.yml @@ -45,7 +45,7 @@ jobs: matrix: name: - debian-10-python-3 - - fedora-32-python-3 + - fedora-33-python-3 - ubuntu-18.04-python-3 - conda-python-3.7-dask-latest - conda-python-3.7-turbodbc-latest @@ -59,11 +59,11 @@ jobs: image: debian-python title: AMD64 Debian 10 Python 3 debian: 10 - - name: fedora-32-python-3 - cache: fedora-32-python-3 + - name: fedora-33-python-3 + cache: fedora-33-python-3 image: fedora-python - title: AMD64 Fedora 32 Python 3 - fedora: 32 + title: AMD64 Fedora 33 Python 3 + fedora: 33 - name: ubuntu-18.04-python-3 cache: ubuntu-18.04-python-3 image: ubuntu-python @@ -102,7 +102,7 @@ jobs: env: # the defaults here should correspond to the values in .env DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 32 }} + FEDORA: ${{ matrix.fedora || 33 }} UBUNTU: ${{ matrix.ubuntu || 18.04 }} PYTHON: ${{ matrix.python || 3.7 }} HDFS: ${{ matrix.hdfs || '2.9.2' }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2481f96db74..15803133fbe 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -187,7 +187,6 @@ jobs: CMAKE_ARGS: >- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} -DBoost_NO_BOOST_CMAKE=ON - CMAKE_GENERATOR: MSYS Makefiles CMAKE_UNITY_BUILD: ON steps: - name: Disable Crash Dialogs diff --git a/.travis.yml b/.travis.yml index 608fb111269..21ac5bc82f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,34 @@ jobs: ARCH: arm64v8 ARROW_CI_MODULES: "CPP" DOCKER_IMAGE_ID: ubuntu-cpp + # ARROW_USE_GLOG=OFF is needed to avoid build error caused by + # glog and CMAKE_UNITY_BUILD=ON. + # + # Disable ARROW_S3 because it often causes "No output has + # been received in the last 10m0s, this potentially indicates + # a stalled build or something wrong with the build itself." + # on Travis CI. + DOCKER_RUN_ARGS: >- + " + -e ARROW_BUILD_STATIC=OFF + -e ARROW_ORC=OFF + -e ARROW_S3=OFF + -e ARROW_USE_GLOG=OFF + -e CMAKE_UNITY_BUILD=ON + " + # We need to use smaller build when cache doesn't exist + # because Travis CI has "No output has been received in the + # last 10m0s" limitation. If we build many modules, we reach + # the limitation. + DOCKER_RUN_ARGS_NO_CACHE: >- + " + -e ARROW_BUILD_TESTS=OFF + -e ARROW_GANDIVA=OFF + -e ARROW_PARQUET=OFF + " + # The LLVM's APT repository provides only arm64 binaries. + # We should use LLVM provided by Ubuntu. + LLVM: "10" UBUNTU: "20.04" - name: "C++ on s390x" os: linux @@ -50,9 +78,25 @@ jobs: ARROW_FLIGHT: "ON" ARROW_PARQUET: "OFF" DOCKER_IMAGE_ID: ubuntu-cpp - PARQUET_BUILD_EXAMPLES: "OFF" - PARQUET_BUILD_EXECUTABLES: "OFF" - Protobuf_SOURCE: "BUNDLED" + # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash. + # Can't enable ARROW_S3 because compiler is killed while compiling + # aws-sdk-cpp. + DOCKER_RUN_ARGS: >- + " + -e ARROW_BUILD_STATIC=OFF + -e ARROW_FLIGHT=ON + -e ARROW_ORC=OFF + -e ARROW_PARQUET=OFF + -e ARROW_S3=OFF + -e PARQUET_BUILD_EXAMPLES=OFF + -e PARQUET_BUILD_EXECUTABLES=OFF + -e Protobuf_SOURCE=BUNDLED + -e cares_SOURCE=BUNDLED + -e gRPC_SOURCE=BUNDLED + " + # The LLVM's APT repository provides only arm64 binaries. + # We should use LLVM provided by Ubuntu. + LLVM: "10" UBUNTU: "20.04" cares_SOURCE: "BUNDLED" gRPC_SOURCE: "BUNDLED" diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml index fa15d77df83..5056456fc66 100644 --- a/ci/conda_env_gandiva.yml +++ b/ci/conda_env_gandiva.yml @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. -clangdev=10 -llvmdev=10 +clangdev=11 +llvmdev=11 re2 diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile similarity index 88% rename from ci/docker/fedora-32-cpp.dockerfile rename to ci/docker/fedora-33-cpp.dockerfile index 766e6b343da..f0c988a5878 100644 --- a/ci/docker/fedora-32-cpp.dockerfile +++ b/ci/docker/fedora-33-cpp.dockerfile @@ -16,7 +16,8 @@ # under the License. ARG arch -FROM ${arch}/fedora:32 +FROM ${arch}/fedora:33 +ARG arch # install dependencies RUN dnf update -y && \ @@ -30,17 +31,18 @@ RUN dnf update -y && \ clang-devel \ cmake \ flatbuffers-devel \ - java-1.8.0-openjdk-devel \ - java-1.8.0-openjdk-headless \ gcc \ gcc-c++ \ - glog-devel \ gflags-devel \ + git \ + glog-devel \ gmock-devel \ google-benchmark-devel \ - protobuf-devel \ + grpc-devel \ + grpc-plugins \ gtest-devel \ - git \ + java-latest-openjdk-devel \ + java-latest-openjdk-headless \ libzstd-devel \ llvm-devel \ llvm-static \ @@ -48,6 +50,7 @@ RUN dnf update -y && \ make \ ninja-build \ openssl-devel \ + protobuf-devel \ python \ rapidjson-devel \ re2-devel \ @@ -56,7 +59,10 @@ RUN dnf update -y && \ which \ zlib-devel -# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck. +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ @@ -75,7 +81,6 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_ZSTD=ON \ CC=gcc \ CXX=g++ \ - gRPC_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_BUILD_EXAMPLES=ON \ diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile index 3d1658ff7b8..957ccb0e630 100644 --- a/ci/docker/linux-apt-c-glib.dockerfile +++ b/ci/docker/linux-apt-c-glib.dockerfile @@ -25,6 +25,7 @@ RUN apt-get update -y -q && \ gtk-doc-tools \ libgirepository1.0-dev \ libglib2.0-doc \ + lsb-release \ luarocks \ pkg-config \ ruby-dev && \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 726a2fdc182..768f13e9772 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -29,7 +29,22 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \ # while debugging package list with docker build. ARG clang_tools ARG llvm -RUN apt-get update -y -q && \ +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ clang-${clang_tools} \ clang-${llvm} \ diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index 9cd999b00e3..412dda48542 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -24,17 +24,20 @@ target=$1 packages=() case "${target}" in cpp|c_glib|ruby) - packages+=(${MINGW_PACKAGE_PREFIX}-ccache) + packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) packages+=(${MINGW_PACKAGE_PREFIX}-boost) packages+=(${MINGW_PACKAGE_PREFIX}-brotli) + packages+=(${MINGW_PACKAGE_PREFIX}-ccache) packages+=(${MINGW_PACKAGE_PREFIX}-clang) packages+=(${MINGW_PACKAGE_PREFIX}-cmake) packages+=(${MINGW_PACKAGE_PREFIX}-gcc) packages+=(${MINGW_PACKAGE_PREFIX}-gflags) packages+=(${MINGW_PACKAGE_PREFIX}-grpc) packages+=(${MINGW_PACKAGE_PREFIX}-gtest) + packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) packages+=(${MINGW_PACKAGE_PREFIX}-llvm) packages+=(${MINGW_PACKAGE_PREFIX}-lz4) + packages+=(${MINGW_PACKAGE_PREFIX}-ninja) packages+=(${MINGW_PACKAGE_PREFIX}-polly) packages+=(${MINGW_PACKAGE_PREFIX}-protobuf) packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8c05d4b84d0..15ac1f4d26e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -92,7 +92,12 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}") -set(ARROW_LLVM_VERSIONS "10" "9" "8" "7") +set(ARROW_LLVM_VERSIONS + "11" + "10" + "9" + "8" + "7") list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY) string(REGEX REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9a842332f94..c6f93419ab1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1706,6 +1706,7 @@ macro(build_rapidjson) set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include") add_dependencies(toolchain rapidjson_ep) + add_dependencies(toolchain-tests rapidjson_ep) add_dependencies(rapidjson rapidjson_ep) set(RAPIDJSON_VENDORED TRUE) diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index ad73751ecdc..cd4edff89ec 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index c9b1a210faa..a5eed2c20a2 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -68,11 +68,9 @@ outputs: - glog - grpc-cpp - libprotobuf - - clangdev 10.* # [not win] - - llvmdev 10.* # [not win] - # llvmdev 9.* or later require Visual Studio 2017 - - clangdev 8.* # [win] - - llvmdev 8.* # [win] + - clangdev 11 + - llvmdev 11 + - libutf8proc - lz4-c - numpy 1.16.* - orc # [unix] diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh index 61c82495af3..50c854689e5 100755 --- a/dev/tasks/gandiva-jars/build-cpp-linux.sh +++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh @@ -63,7 +63,7 @@ cmake -DCMAKE_BUILD_TYPE=Release \ -DBOOST_ROOT=/arrow_boost_dist \ -GNinja /arrow/cpp ninja install -ninja test +CTEST_OUTPUT_ON_FAILURE=1 ninja test popd diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index c5dbb526c8b..e94d94997b0 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -15,7 +15,7 @@ class ApacheArrow < Formula depends_on "numpy" depends_on "openssl@1.1" depends_on "protobuf" - depends_on "python" + depends_on "python@3.9" depends_on "rapidjson" depends_on "snappy" depends_on "thrift" @@ -38,7 +38,7 @@ def install -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_INSTALL_NAME_RPATH=OFF - -DPYTHON_EXECUTABLE=#{Formula["python"].bin/"python3"} + -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"} ] # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437 diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile index 3b1b3eb3987..a26b75c892a 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -33,6 +33,15 @@ ARG LLVM RUN \ quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ apt update ${quiet} && \ + apt install -y -V ${quiet} \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + apt update ${quiet} && \ apt install -y -V ${quiet} \ build-essential \ cmake \ diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 2ea6845ed28..d3c36aa79a2 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -54,7 +54,7 @@ def initialize(package, version, release_time, options={}) @rpm_version = @version @rpm_release = "1" end - @deb_release = "1" + @deb_release = ENV["DEB_RELEASE"] || "1" end def define diff --git a/dev/tasks/nuget-packages/github.linux.yml b/dev/tasks/nuget-packages/github.linux.yml index c8143e432c1..95c4464355e 100644 --- a/dev/tasks/nuget-packages/github.linux.yml +++ b/dev/tasks/nuget-packages/github.linux.yml @@ -40,7 +40,9 @@ jobs: git -C arrow checkout FETCH_HEAD git -C arrow submodule update --init --recursive - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 + with: + python-version: 3.8 - name: Setup Archery run: pip install -e arrow/dev/archery[docker] - name: Prepare version diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e8de06415c5..941f0b394a1 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1585,12 +1585,12 @@ tasks: UBUNTU: 18.04 run: ubuntu-cpp - test-fedora-32-cpp: + test-fedora-33-cpp: ci: circle template: docker-tests/circle.linux.yml params: env: - FEDORA: 32 + FEDORA: 33 run: fedora-cpp test-ubuntu-18.04-cpp-release: @@ -1693,12 +1693,12 @@ tasks: UBUNTU: 18.04 run: ubuntu-python - test-fedora-32-python-3: + test-fedora-33-python-3: ci: azure template: docker-tests/azure.linux.yml params: env: - FEDORA: 32 + FEDORA: 33 run: fedora-python test-r-linux-as-cran: diff --git a/docker-compose.yml b/docker-compose.yml index 9d838ef5613..da57c69e0c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -321,7 +321,7 @@ services: # docker-compose run --rm fedora-cpp # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 32 + # FEDORA: 33 image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp build: context: . @@ -560,7 +560,7 @@ services: # docker-compose run --rm fedora-python # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 32 + # FEDORA: 33 image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 build: context: .