ARROW-12262: [Doc] Enable S3 and Flight in docs build

.env

@@ -42,7 +42,7 @@ ULIMIT_CORE=-1
 REPO=apache/arrow-dev
 CUDA=9.1
 DEBIAN=10
-UBUNTU=18.04
+UBUNTU=20.04
 FEDORA=33
 PYTHON=3.6
 LLVM=11

.github/workflows/cpp.yml

@@ -59,7 +59,7 @@ jobs:
           - image: conda-cpp
             title: AMD64 Conda C++
           - image: ubuntu-cpp-sanitizer
-            title: AMD64 Ubuntu 18.04 C++ ASAN UBSAN
+            title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2

ci/docker/linux-apt-docs.dockerfile

@@ -18,9 +18,10 @@
 ARG base
 FROM ${base}
 
-ARG r=3.6
+ARG r=4.0
 ARG jdk=8
 
+# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/
 RUN apt-get update -y && \
     apt-get install -y \
         dirmngr \
@@ -29,8 +30,8 @@ RUN apt-get update -y && \
     apt-key adv \
         --keyserver keyserver.ubuntu.com \
         --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
-    add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran35/' && \
-    apt-get install -y \
+    add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \
+    apt-get install -y --no-install-recommends \
         autoconf-archive \
         automake \
         curl \
@@ -43,6 +44,7 @@ RUN apt-get update -y && \
         libgirepository1.0-dev \
         libglib2.0-doc \
         libharfbuzz-dev \
+        libtiff-dev \
         libtool \
         libxml2-dev \
         ninja-build \
@@ -72,7 +74,6 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \
     rm -rf /var/lib/apt/lists/* && \
     npm install -g yarn
 
-# Sphinx is pinned because of ARROW-9693
 RUN pip install \
         meson \
         breathe \
@@ -96,8 +97,11 @@ COPY r/DESCRIPTION /arrow/r/
 RUN /arrow/ci/scripts/r_deps.sh /arrow && \
     R -e "install.packages('pkgdown')"
 
-ENV ARROW_PYTHON=ON \
+ENV ARROW_FLIGHT=ON \
+    ARROW_PYTHON=ON \
+    ARROW_S3=ON \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_USE_GLOG=OFF \
+    CMAKE_UNITY_BUILD=ON \

ci/docker/ubuntu-20.10-cpp.dockerfile

@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base=amd64/ubuntu:20.10
+FROM ${base}
+ARG arch
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN echo "debconf debconf/frontend select Noninteractive" | \
+        debconf-set-selections
+
+# Installs LLVM toolchain, for Gandiva and testing other compilers
+#
+# Note that this is installed before the base packages to improve iteration
+# while debugging package list with docker build.
+ARG clang_tools
+ARG llvm
+RUN if [ "${llvm}" -gt "10" ]; then \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          apt-transport-https \
+          ca-certificates \
+          gnupg \
+          wget && \
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+      echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \
+         /etc/apt/sources.list.d/llvm.list && \
+      if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+        echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \
+           /etc/apt/sources.list.d/clang-tools.list; \
+      fi \
+    fi && \
+    apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        clang-${clang_tools} \
+        clang-${llvm} \
+        clang-format-${clang_tools} \
+        clang-tidy-${clang_tools} \
+        llvm-${llvm}-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+# Installs C++ toolchain and dependencies
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        autoconf \
+        ca-certificates \
+        ccache \
+        cmake \
+        g++ \
+        gcc \
+        gdb \
+        git \
+        libbenchmark-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
+        libbrotli-dev \
+        libbz2-dev \
+        libgflags-dev \
+        libcurl4-openssl-dev \
+        libgoogle-glog-dev \
+        libgrpc++-dev \
+        liblz4-dev \
+        libprotobuf-dev \
+        libprotoc-dev \
+        libre2-dev \
+        libsnappy-dev \
+        libssl-dev \
+        libthrift-dev \
+        libutf8proc-dev \
+        libzstd-dev \
+        make \
+        ninja-build \
+        pkg-config \
+        protobuf-compiler \
+        protobuf-compiler-grpc \
+        rapidjson-dev \
+        tzdata \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
+# Prioritize system packages and local installation
+# The following dependencies will be downloaded due to missing/invalid packages
+# provided by the distribution:
+# - libc-ares-dev does not install CMake config files
+# - flatbuffer is not packaged
+# - libgtest-dev only provide sources
+# - libprotobuf-dev only provide sources
+ENV ARROW_BUILD_TESTS=ON \
+    ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_DATASET=ON \
+    ARROW_FLIGHT=OFF \
+    ARROW_GANDIVA=ON \
+    ARROW_HDFS=ON \
+    ARROW_HOME=/usr/local \
+    ARROW_INSTALL_NAME_RPATH=OFF \
+    ARROW_NO_DEPRECATED_API=ON \
+    ARROW_ORC=ON \
+    ARROW_PARQUET=ON \
+    ARROW_PLASMA=ON \
+    ARROW_S3=ON \
+    ARROW_USE_ASAN=OFF \
+    ARROW_USE_CCACHE=ON \
+    ARROW_USE_UBSAN=OFF \
+    ARROW_WITH_BROTLI=ON \
+    ARROW_WITH_BZ2=ON \
+    ARROW_WITH_LZ4=ON \
+    ARROW_WITH_SNAPPY=ON \
+    ARROW_WITH_ZLIB=ON \
+    ARROW_WITH_ZSTD=ON \
+    AWSSDK_SOURCE=BUNDLED \
+    GTest_SOURCE=BUNDLED \
+    ORC_SOURCE=BUNDLED \
+    PARQUET_BUILD_EXAMPLES=ON \
+    PARQUET_BUILD_EXECUTABLES=ON \
+    PATH=/usr/lib/ccache/:$PATH \
+    PYTHON=python3

ci/scripts/docs_build.sh

@@ -27,8 +27,10 @@ export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
 export CFLAGS="-DARROW_NO_DEPRECATED_API"
 export CXXFLAGS="-DARROW_NO_DEPRECATED_API"
 
-# Prose and Python
-sphinx-build -b html ${arrow_dir}/docs/source ${build_dir}
+ncpus=$(python3 -c "import os; print(os.cpu_count())")
+
+# Sphinx docs
+sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir}
 
 # C++ - original doxygen
 # rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp

dev/release/post-09-docs.sh

@@ -46,6 +46,7 @@ git checkout "${release_tag}"
 archery docker run \
   -v "${ARROW_SITE_DIR}/docs:/build/docs" \
   -e ARROW_DOCS_VERSION="${version}" \
+  -e UBUNTU=20.10 \
   ubuntu-docs
 
 : ${PUSH:=1}

dev/tasks/tasks.yml

@@ -674,6 +674,8 @@ tasks:
     params:
       target: "ubuntu-bionic"
       task_namespace: "apt"
+      env:
+        UBUNTU: 18.04
       upload_extensions:
         - .ddeb
         - .deb
@@ -1474,6 +1476,8 @@ tasks:
     ci: azure
     template: docker-tests/azure.linux.yml
     params:
+      env:
+        UBUNTU: 18.04
       run: ubuntu-r-sanitizer
 
   test-debian-10-go-1.15:
@@ -1484,12 +1488,12 @@ tasks:
         GO: 1.15
       run: debian-go
 
-  test-ubuntu-18.04-docs:
+  test-ubuntu-20.10-docs:
     ci: azure
     template: docker-tests/azure.linux.yml
     params:
       env:
-        UBUNTU: 18.04
+        UBUNTU: "20.10"
       run: ubuntu-docs
 
   ############################## vcpkg tests ##################################

docker-compose.yml

@@ -366,6 +366,7 @@ services:
       ARROW_FUZZING: "ON"  # Check fuzz regressions
       ARROW_JEMALLOC: "OFF"
       ARROW_ORC: "OFF"
+      ARROW_S3: "OFF"
       ARROW_USE_ASAN: "ON"
       ARROW_USE_UBSAN: "ON"
       # utf8proc 2.1.0 in Ubuntu Bionic has test failures

docs/source/cpp/examples/row_columnar_conversion.rst

@@ -24,4 +24,4 @@ Row to columnar conversion
 The following example converts an array of structs to a :class:`arrow::Table`
 instance, and then converts it back to the original array of structs.
 
-.. literalinclude:: ../../../../cpp/examples/arrow/row-wise-conversion-example.cc
+.. literalinclude:: ../../../../cpp/examples/arrow/row_wise_conversion_example.cc

docs/source/status.rst

@@ -126,7 +126,7 @@ IPC Format
 +-----------------------------+-------+-------+-------+------------+-------+-------+-------+
 | Sparse tensors              | ✓     |       |       |            |       |       |       |
 +-----------------------------+-------+-------+-------+------------+-------+-------+-------+
-| Buffer compression          | ✓     | ✓ (3) | ✓    |            |       |       | ✓     |
+| Buffer compression          | ✓     | ✓ (3) | ✓     |            |       |       | ✓     |
 +-----------------------------+-------+-------+-------+------------+-------+-------+-------+
 | Endianness conversion       | ✓ (2) |       |       |            |       |       |       |
 +-----------------------------+-------+-------+-------+------------+-------+-------+-------+