Alamb/update docs

.dockerignore

@@ -55,6 +55,8 @@
 !rust/arrow-flight/Cargo.toml
 !rust/parquet/Cargo.toml
 !rust/parquet/build.rs
+!rust/parquet_derive/Cargo.toml
+!rust/parquet_derive_test/Cargo.toml
 !rust/datafusion/Cargo.toml
 !rust/datafusion/benches
 !rust/integration-testing/Cargo.toml

.env

@@ -24,13 +24,13 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=32
+FEDORA=33
 PYTHON=3.6
-LLVM=10
+LLVM=11
 CLANG_TOOLS=8
 RUST=nightly-2020-04-22
 GO=1.12
-NODE=11
+NODE=14
 MAVEN=3.5.4
 JDK=8
 PANDAS=latest

.github/workflows/archery.yml

@@ -42,9 +42,10 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Fetch Submodules and Tags
+      - name: Git Fixup
+        if: ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: ci/scripts/util_checkout.sh
+        run: git branch master origin/master
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Setup Python

.github/workflows/comment_bot.yml

@@ -69,7 +69,7 @@ jobs:
           git remote add upstream https://github.com/apache/arrow
           git fetch upstream
           changed() {
-            git diff --name-only HEAD^..upstream/master | grep -e "$1" >/dev/null 2>&1
+            git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1
           }
           if changed '^r/.*\.R$'; then
             echo "::set-env name=R_DOCS::true"
@@ -84,10 +84,11 @@ jobs:
             echo "::set-env name=CLANG_FORMAT_R::true"
           fi
       - name: Run cmake_format
-        if: false
-        # TODO: make this work https://issues.apache.org/jira/browse/ARROW-8489
-        # if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything')
+        if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything')
         run: |
+          set -ex
+          export PATH=/home/runner/.local/bin:$PATH
+          python3 -m pip install --upgrade pip setuptools wheel
           python3 -m pip install -r dev/archery/requirements-lint.txt
           python3 run-cmake-format.py
       - name: Run clang-format on cpp

.github/workflows/cpp.yml

@@ -102,9 +102,11 @@ jobs:
     #         hosted machines
     name: ${{ matrix.title }}
     runs-on: ${{ matrix.runner }}
-    if: github.event_name == 'push'
+    # TODO(kszucs): re-enable once the self-hosted workers are properly
+    # registered to github
+    if: false && github.event_name == 'push'
     defaults:
-      # to use certain environment variables are set by .bashrc an interactive
+      # To use certain environment variables set by .bashrc, an interactive
       # bash shell must be used
       run:
         shell: bash -i {0}
@@ -119,11 +121,12 @@ jobs:
             debian: 10
             title: ARM32v7 Debian 10 C++
             image: |
-              -e CPP_MAKE_PARALLELISM=4 \
+              -e CPP_MAKE_PARALLELISM=2 \
               -e CXXFLAGS=-Wno-psabi \
               -e ARROW_PARQUET=OFF \
               -e ARROW_FLIGHT=OFF \
               -e ARROW_GANDIVA=OFF \
+              -e ARROW_ORC=OFF \
               -e CMAKE_ARGS=-DARROW_CPU_FLAG=armv7 \
               debian-cpp
             arch: 'arm32v7'
@@ -186,23 +189,24 @@ jobs:
     strategy:
       fail-fast: false
     env:
+      ARROW_BUILD_TESTS: ON
+      ARROW_DATASET: ON
+      ARROW_FLIGHT: ON
+      ARROW_GANDIVA: ON
+      ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
       # TODO(kszucs): link error in the tests
-      ARROW_DATASET: ON
       ARROW_ORC: OFF
-      ARROW_FLIGHT: ON
-      ARROW_HDFS: ON
-      ARROW_PLASMA: ON
-      ARROW_GANDIVA: ON
       ARROW_PARQUET: ON
-      ARROW_WITH_ZLIB: ON
-      ARROW_WITH_LZ4: ON
+      ARROW_PLASMA: ON
+      ARROW_S3: ON
+      ARROW_WITH_BROTLI: ON
       ARROW_WITH_BZ2: ON
-      ARROW_WITH_ZSTD: ON
+      ARROW_WITH_LZ4: ON
       ARROW_WITH_SNAPPY: ON
-      ARROW_WITH_BROTLI: ON
-      ARROW_BUILD_TESTS: ON
+      ARROW_WITH_ZLIB: ON
+      ARROW_WITH_ZSTD: ON
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -319,6 +323,7 @@ jobs:
       ARROW_JEMALLOC: OFF
       ARROW_PARQUET: ON
       ARROW_PYTHON: ON
+      ARROW_S3: ON
       ARROW_USE_GLOG: OFF
       ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
       ARROW_WITH_BROTLI: ON
@@ -333,7 +338,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs
@@ -370,6 +374,14 @@ jobs:
         run: |
           export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
           ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Download MinIO
+        shell: msys2 {0}
+        run: |
+          mkdir -p /usr/local/bin
+          wget \
+            --output-document /usr/local/bin/minio.exe \
+            https://dl.min.io/server/minio/release/windows-amd64/minio.exe
+          chmod +x /usr/local/bin/minio.exe
       - name: Test
         shell: msys2 {0}
         run: |

.github/workflows/cpp_cron.yml

@@ -46,7 +46,7 @@ jobs:
       matrix:
         name:
           - amd64-debian-10-cpp
-          - amd64-fedora-32-cpp
+          - amd64-fedora-33-cpp
           - amd64-ubuntu-16.04-cpp
           - amd64-ubuntu-18.04-cpp
           - amd64-ubuntu-18.04-cpp-cmake32
@@ -55,10 +55,10 @@ jobs:
             image: debian-cpp
             title: AMD64 Debian 10 C++
             debian: 10
-          - name: amd64-fedora-32-cpp
+          - name: amd64-fedora-33-cpp
             image: fedora-cpp
-            title: AMD64 Fedora 32 C++
-            fedora: 32
+            title: AMD64 Fedora 33 C++
+            fedora: 33
           - name: amd64-ubuntu-16.04-cpp
             image: ubuntu-cpp
             title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
       # the defaults here should correspond to the values in .env
       ARCH: 'amd64'
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
     steps:
       - name: Checkout Arrow

.github/workflows/dev.yml

@@ -59,43 +59,6 @@ jobs:
         continue-on-error: true
         run: archery docker push ubuntu-lint
 
-  docs:
-    name: Sphinx and API documentations
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    steps:
-      - name: Checkout Arrow
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Fetch Submodules and Tags
-        shell: bash
-        run: ci/scripts/util_checkout.sh
-      - name: Free Up Disk Space
-        shell: bash
-        run: ci/scripts/util_cleanup.sh
-      - name: Cache Docker Volumes
-        uses: actions/cache@v1
-        with:
-          path: .docker
-          key: ubuntu-18.04-${{ hashFiles('cpp/**') }}
-          restore-keys: ubuntu-18.04-
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Setup Archery
-        run: pip install -e dev/archery[docker]
-      - name: Execute Docker Build
-        run: |
-          sudo sysctl -w kernel.core_pattern="core.%e.%p"
-          ulimit -c unlimited
-          archery docker run ubuntu-docs
-      - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
-        continue-on-error: true
-        run: archery docker push ubuntu-docs
-
   release:
     name: Source Release and Merge Script
     runs-on: ubuntu-latest

.github/workflows/dev_labeler.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PR labeler
+on:
+  pull_request_target:
+    types: [opened, reopened]
+    paths:
+      - 'rust/**'
+
+jobs:
+  assign-rust-labels:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Assign Github labels
+      uses: actions/[email protected]
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        configuration-path: .github/workflows/dev_labeler/labeler.yml
+        sync-labels: true

.github/workflows/dev_labeler/labeler.yml

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+lang-rust:
+  - rust/**/*
+
+datafusion:
+  - rust/datafusion/**/*

.github/workflows/java.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
-        run: archery docker push debian-go
+        run: archery docker push debian-java
 
   macos:
     name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}

.github/workflows/js.yml

@@ -40,7 +40,7 @@ env:
 jobs:
 
   docker:
-    name: AMD64 Debian 10 NodeJS 11
+    name: AMD64 Debian 10 NodeJS 14
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     steps:
@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node: [11]
+        node: [14]
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -103,7 +103,7 @@ jobs:
   #   strategy:
   #     fail-fast: false
   #     matrix:
-  #       node: [11]
+  #       node: [14]
   #   steps:
   #     - name: Checkout Arrow
   #       uses: actions/checkout@v1

.github/workflows/python.yml

@@ -50,7 +50,7 @@ jobs:
           - ubuntu-16.04-python-3
           - conda-python-3.8-nopandas
           - conda-python-3.6-pandas-0.23
-          - conda-python-3.6-pandas-latest
+          - conda-python-3.7-pandas-latest
           - centos-python-3.6-manylinux1
         include:
           - name: ubuntu-16.04-python-3
@@ -70,11 +70,11 @@ jobs:
             title: AMD64 Conda Python 3.6 Pandas 0.23
             python: 3.6
             pandas: 0.23
-          - name: conda-python-3.6-pandas-latest
-            cache: conda-python-3.6
+          - name: conda-python-3.7-pandas-latest
+            cache: conda-python-3.7
             image: conda-python-pandas
-            title: AMD64 Conda Python 3.6 Pandas latest
-            python: 3.6
+            title: AMD64 Conda Python 3.7 Pandas latest
+            python: 3.7
             pandas: latest
           - name: centos-python-3.6-manylinux1
             cache: manylinux1
@@ -116,7 +116,7 @@ jobs:
         run: archery docker push ${{ matrix.image }}
 
   macos:
-    name: AMD64 MacOS 10.15 Python 3.7
+    name: AMD64 MacOS 10.15 Python 3
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     env:
@@ -150,8 +150,9 @@ jobs:
           brew update --preinstall
           brew bundle --file=cpp/Brewfile
           brew install coreutils python
-          pip3 install -r python/requirements-build.txt \
-                       -r python/requirements-test.txt
+          python3 -mpip install \
+            -r python/requirements-build.txt \
+            -r python/requirements-test.txt
       - name: Build
         shell: bash
         run: |