diff --git a/.env b/.env index cd6b57e004a..c4eb6d2de43 100644 --- a/.env +++ b/.env @@ -42,7 +42,7 @@ ULIMIT_CORE=-1 REPO=apache/arrow-dev CUDA=9.1 DEBIAN=10 -UBUNTU=18.04 +UBUNTU=20.04 FEDORA=33 PYTHON=3.6 LLVM=11 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 5f25deb4512..0bcf3460ad4 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -59,7 +59,7 @@ jobs: - image: conda-cpp title: AMD64 Conda C++ - image: ubuntu-cpp-sanitizer - title: AMD64 Ubuntu 18.04 C++ ASAN UBSAN + title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 604a05afb07..46c31bbd480 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -18,9 +18,10 @@ ARG base FROM ${base} -ARG r=3.6 +ARG r=4.0 ARG jdk=8 +# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/ RUN apt-get update -y && \ apt-get install -y \ dirmngr \ @@ -29,8 +30,8 @@ RUN apt-get update -y && \ apt-key adv \ --keyserver keyserver.ubuntu.com \ --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran35/' && \ - apt-get install -y \ + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \ + apt-get install -y --no-install-recommends \ autoconf-archive \ automake \ curl \ @@ -43,6 +44,7 @@ RUN apt-get update -y && \ libgirepository1.0-dev \ libglib2.0-doc \ libharfbuzz-dev \ + libtiff-dev \ libtool \ libxml2-dev \ ninja-build \ @@ -72,7 +74,6 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ rm -rf /var/lib/apt/lists/* && \ npm install -g yarn -# Sphinx is pinned because of ARROW-9693 RUN pip install \ meson \ breathe \ @@ -96,8 +97,11 @@ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow && \ R -e "install.packages('pkgdown')" -ENV ARROW_PYTHON=ON \ +ENV ARROW_FLIGHT=ON \ + ARROW_PYTHON=ON \ + ARROW_S3=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ ARROW_USE_GLOG=OFF \ + CMAKE_UNITY_BUILD=ON \ diff --git a/ci/docker/ubuntu-20.10-cpp.dockerfile b/ci/docker/ubuntu-20.10-cpp.dockerfile new file mode 100644 index 00000000000..80eb072e7ed --- /dev/null +++ b/ci/docker/ubuntu-20.10-cpp.dockerfile @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.10 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libgflags-dev \ + libcurl4-openssl-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + rapidjson-dev \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh index a0d926a335e..e6ee768ee87 100755 --- a/ci/scripts/docs_build.sh +++ b/ci/scripts/docs_build.sh @@ -27,8 +27,10 @@ export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 export CFLAGS="-DARROW_NO_DEPRECATED_API" export CXXFLAGS="-DARROW_NO_DEPRECATED_API" -# Prose and Python -sphinx-build -b html ${arrow_dir}/docs/source ${build_dir} +ncpus=$(python3 -c "import os; print(os.cpu_count())") + +# Sphinx docs +sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir} # C++ - original doxygen # rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh index 51e74541eb7..c9f75b48b2c 100755 --- a/dev/release/post-09-docs.sh +++ b/dev/release/post-09-docs.sh @@ -46,6 +46,7 @@ git checkout "${release_tag}" archery docker run \ -v "${ARROW_SITE_DIR}/docs:/build/docs" \ -e ARROW_DOCS_VERSION="${version}" \ + -e UBUNTU=20.10 \ ubuntu-docs : ${PUSH:=1} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index dcf49414fcc..0080b387663 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -674,6 +674,8 @@ tasks: params: target: "ubuntu-bionic" task_namespace: "apt" + env: + UBUNTU: 18.04 upload_extensions: - .ddeb - .deb @@ -1474,6 +1476,8 @@ tasks: ci: azure template: docker-tests/azure.linux.yml params: + env: + UBUNTU: 18.04 run: ubuntu-r-sanitizer test-debian-10-go-1.15: @@ -1484,12 +1488,12 @@ tasks: GO: 1.15 run: debian-go - test-ubuntu-18.04-docs: + test-ubuntu-20.10-docs: ci: azure template: docker-tests/azure.linux.yml params: env: - UBUNTU: 18.04 + UBUNTU: "20.10" run: ubuntu-docs ############################## vcpkg tests ################################## diff --git a/docker-compose.yml b/docker-compose.yml index 539d5adcb97..b22b1f6b536 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -366,6 +366,7 @@ services: ARROW_FUZZING: "ON" # Check fuzz regressions ARROW_JEMALLOC: "OFF" ARROW_ORC: "OFF" + ARROW_S3: "OFF" ARROW_USE_ASAN: "ON" ARROW_USE_UBSAN: "ON" # utf8proc 2.1.0 in Ubuntu Bionic has test failures diff --git a/docs/source/cpp/examples/row_columnar_conversion.rst b/docs/source/cpp/examples/row_columnar_conversion.rst index 02fd61b50c7..3f45864c228 100644 --- a/docs/source/cpp/examples/row_columnar_conversion.rst +++ b/docs/source/cpp/examples/row_columnar_conversion.rst @@ -24,4 +24,4 @@ Row to columnar conversion The following example converts an array of structs to a :class:`arrow::Table` instance, and then converts it back to the original array of structs. -.. literalinclude:: ../../../../cpp/examples/arrow/row-wise-conversion-example.cc +.. literalinclude:: ../../../../cpp/examples/arrow/row_wise_conversion_example.cc diff --git a/docs/source/status.rst b/docs/source/status.rst index acf5af90d52..176d35eb12b 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -126,7 +126,7 @@ IPC Format +-----------------------------+-------+-------+-------+------------+-------+-------+-------+ | Sparse tensors | ✓ | | | | | | | +-----------------------------+-------+-------+-------+------------+-------+-------+-------+ -| Buffer compression | ✓ | ✓ (3) | ✓ | | | | ✓ | +| Buffer compression | ✓ | ✓ (3) | ✓ | | | | ✓ | +-----------------------------+-------+-------+-------+------------+-------+-------+-------+ | Endianness conversion | ✓ (2) | | | | | | | +-----------------------------+-------+-------+-------+------------+-------+-------+-------+